Compare commits

...

143 Commits

Author SHA1 Message Date
fengmk2
994b2e47aa Release 0.8.0 2014-07-21 13:58:24 +08:00
dead_horse
b14faad3fb Merge pull request #375 from cnpm/scope-package
support scoped package
2014-07-21 13:56:13 +08:00
fengmk2
9163f793a3 update readme show scoped packages 2014-07-21 13:53:33 +08:00
fengmk2
1c0d4fdcdc support "scoped" packages. close #352 2014-07-21 13:51:43 +08:00
dead_horse
20fce9f55f Merge pull request #373 from cnpm/stop-old-publish-flow
Stop old publish flow
2014-07-19 01:21:22 +08:00
fengmk2
6b73a7762f use safe jsonp 2014-07-19 01:16:30 +08:00
fengmk2
fd34e0512c Stop support old publish flow. fix #368 2014-07-19 01:11:33 +08:00
fengmk2
a84715167e Merge pull request #372 from cnpm/sql
update SQLs
2014-07-13 07:07:59 +08:00
dead_horse
db1cb0d0f8 update SQLs 2014-07-12 23:20:55 +08:00
dead_horse
dc137f0821 bump redis 2014-07-12 14:57:23 +08:00
fengmk2
76d270fc77 Merge pull request #371 from cnpm/update-logger
use sync_info and sync_error categories
2014-07-11 12:06:28 +08:00
dead_horse
b5190b56e8 use sync_info and sync_error categories 2014-07-11 10:03:41 +08:00
fengmk2
d2d238db59 add categories to loggers. fix #370 2014-07-11 09:07:48 +08:00
fengmk2
4c02d84fec fix get latest tag always not exists bug 2014-07-11 01:04:52 +08:00
dead_horse
74585f0b8a Merge pull request #369 from cnpm/publish-with-tag
support `npm publish --tag beta`. fix #366
2014-07-10 23:47:48 +08:00
fengmk2
a6ac632e73 support npm publish --tag beta. fix #366 2014-07-10 23:34:25 +08:00
fengmk2
0b59547ac1 Merge pull request #365 from cnpm/logger
use mini-logger and error-formater
2014-07-09 11:30:48 +08:00
dead_horse
b5929c94e0 use mini-logger and error-formater 2014-07-09 11:23:20 +08:00
fengmk2
4c96682603 Release 0.7.0 2014-07-07 12:11:12 +08:00
dead_horse
78af0fc15b Merge pull request #364 from cnpm/maintainer
Maintainer logic improve
2014-07-07 11:42:53 +08:00
fengmk2
ce381618a0 use module_maintainers on GET /pakcage/:name page 2014-07-07 10:25:06 +08:00
fengmk2
87486c484f use new module_maintainers on GET /:name 2014-07-07 10:23:53 +08:00
fengmk2
4608712e05 admin user should never publish to other user's packages. fix #363
but admin user need have the permission to delete other's packages.
2014-07-07 01:41:03 +08:00
fengmk2
3c6576bcab Add a new table for module-maintainers.
* fix #362 update maintianers change the lastmodified. remove version and change last modified.
* #363 change isMaintainer detect logic.
2014-07-07 00:56:44 +08:00
fengmk2
ba1986b931 gravatar use https 2014-07-02 00:09:56 +08:00
fengmk2
5efa508376 support https 2014-07-01 23:26:48 +08:00
dead_horse
38617d4572 bump dependencies 2014-06-30 17:31:04 +08:00
fengmk2
a9a04b370d Release 0.6.1 2014-06-18 12:27:47 +08:00
fengmk2
ec1d924dc1 hot fix removeTagsByNames() 2014-06-17 10:41:41 +08:00
fengmk2
234d1ec4d6 fix _rev not exists 2014-06-16 15:52:56 +08:00
dead_horse
b8f4958d8f Merge pull request #357 from cnpm/fix-sync-request
sync unpublished on GET /sync/:name
2014-06-16 15:23:22 +08:00
fengmk2
e56a21e890 sync unpublished on GET /sync/:name 2014-06-16 15:21:53 +08:00
fengmk2
e3434a58c5 Release 0.6.0 2014-06-16 15:06:17 +08:00
dead_horse
cd9d38aadb Merge pull request #356 from cnpm/unpublished
sync unpublished info. close #353
2014-06-16 13:50:05 +08:00
fengmk2
0ff22ccc60 sync unpublished info. close #353 2014-06-16 10:54:02 +08:00
dead_horse
e4cfb13383 Merge pull request #354 from cnpm/delete-not-exists-versions
Delete not exists versions on sync worker. #353
2014-06-16 09:39:58 +08:00
fengmk2
23ad3a29b4 Delete not exists versions on sync worker. #353 2014-06-15 00:57:18 +08:00
dead_horse
f11b4e9954 Release 0.5.3 2014-06-13 18:08:12 +08:00
dead_horse
6e775eed47 fix sync response 204 2014-06-13 18:05:09 +08:00
dead_horse
ca6841c042 add links in History.md 2014-06-08 00:59:54 +08:00
dead_horse
652b900fc4 bump koa 2014-06-08 00:59:22 +08:00
dead_horse
43379cc66d fix test-cov 2014-06-04 17:37:19 +08:00
dead_horse
8f9b3db029 bump koa and should 2014-06-04 17:29:52 +08:00
fengmk2
884b75f6a3 Release 0.5.2 2014-06-04 15:41:01 +08:00
dead_horse
a958ee40c4 Merge pull request #350 from cnpm/hotfix
sync hotfix
2014-06-04 15:39:05 +08:00
fengmk2
1a01909750 sync hotfix 2014-06-04 15:35:50 +08:00
dead_horse
6901a5c994 Merge pull request #349 from cnpm/dist-sync-phantomjs
sync phantomjs downloads pkg. close #348
2014-06-04 15:12:31 +08:00
fengmk2
33b13dc30e sync phantomjs downloads pkg. close #348 2014-06-04 14:35:20 +08:00
fengmk2
73d9d2c3d0 Merge pull request #347 from cnpm/issue346-restart-syncer
add restart, fixed #346
2014-06-04 12:39:34 +08:00
dead_horse
8b955adb42 add restart, fixed #346 2014-06-04 10:57:50 +08:00
fengmk2
1f219cbd1b Release 0.5.1 2014-05-28 09:18:15 +08:00
dead_horse
17025d42b4 Merge pull request #344 from cnpm/list-since-from-tag
fix attack on /-/all/since?stale=update_after&startkey=2 close #336
2014-05-28 09:09:43 +08:00
fengmk2
922f26b052 fix attack on /-/all/since?stale=update_after&startkey=2 close #336
Also add modified_time key to tag
2014-05-28 00:19:26 +08:00
dead_horse
9371685188 bump thunkify-wrap 2014-05-26 21:44:51 +08:00
dead_horse
ad0b66c7e5 bump koa-middlewares 2014-05-25 01:03:19 +08:00
dead_horse
5588880ec0 remove outputError 2014-05-25 00:44:26 +08:00
dead_horse
9ad72cbdcb bump dependencies 2014-05-25 00:43:16 +08:00
dead_horse
4d033471fc use svg badge 2014-05-25 00:36:58 +08:00
fengmk2
590b7f7eee Merge pull request #340 from cnpm/404-page
add package/notfound page
2014-05-19 19:26:34 +08:00
dead_horse
0ce817f9f3 add package/notfound page 2014-05-19 19:16:15 +08:00
fengmk2
59af2bfd56 add dist mirror link to home page 2014-05-13 09:37:46 +08:00
fengmk2
a83f6bb183 fix sync listdiff and add more test cases 2014-05-13 09:28:54 +08:00
fengmk2
b6f1531743 Release 0.5.0 2014-05-13 08:32:51 +08:00
dead_horse
42d0b538ca Merge pull request #337 from cnpm/dist-sync
Dist sync
2014-05-12 23:48:52 +08:00
fengmk2
a240eb9922 filter /nightlies/* 2014-05-12 17:32:06 +08:00
fengmk2
6853b73fb2 use koa setter instead of set() 2014-05-12 17:09:51 +08:00
fengmk2
a887acec05 update co-urllib 2014-05-12 10:19:19 +08:00
fengmk2
1d3f0f0ad3 add more info on error email 2014-05-12 10:03:01 +08:00
fengmk2
20e04065df add sync dist to sync/index.js 2014-05-12 07:38:48 +08:00
fengmk2
1cee298bc3 show dist page 2014-05-11 20:56:40 +08:00
fengmk2
854bc6c9d8 sync dist file and save it to database 2014-05-10 18:59:13 +08:00
fengmk2
b70c1c421a disable gzip before #335 has fix 2014-05-04 20:05:55 +08:00
dead_horse
b1b6172892 Release 0.4.3 2014-04-18 16:15:50 +08:00
fengmk2
e5a77a4368 Merge pull request #334 from cnpm/fix-permission
add permission check to /:name/:tag
2014-04-18 14:34:27 +08:00
dead_horse
b74cccd342 add permission check to /:name/:tag 2014-04-18 14:29:42 +08:00
fengmk2
6aae538f49 Merge pull request #333 from cnpm/issue332-tag
add put /:name/:tag, close #332
2014-04-18 14:12:56 +08:00
dead_horse
74c8b25374 fix space 2014-04-18 14:07:49 +08:00
dead_horse
0cec2edea6 add put /:name/:tag, close #332 2014-04-18 12:18:07 +08:00
fengmk2
b59a0a99cf Release 0.4.2 2014-04-17 17:33:17 +08:00
dead_horse
c7e82809e3 Merge pull request #330 from cnpm/package-size-show
fix fav ico and show pkg size on pkg info page. fix #318
2014-04-17 17:31:06 +08:00
fengmk2
206ee505a8 sync interval config 2014-04-17 17:28:27 +08:00
fengmk2
d39838f930 fix fav ico and show pkg size on pkg info page. fix #318 2014-04-17 14:59:38 +08:00
dead_horse
512e21aaf5 Merge pull request #329 from cnpm/fix-stack-over
sync work sync one done must wait for a defer.setImmediate. fix #328
2014-04-17 14:12:22 +08:00
fengmk2
f7904ee699 sync work sync one done must wait for a defer.setImmediate. fix #328 2014-04-17 09:11:46 +08:00
dead_horse
7007be6ef4 bump dep versions 2014-04-16 15:17:14 +08:00
dead_horse
70aa3299e5 Merge pull request #326 from cnpm/noattachment
if download tarball 404, throw err better than ignore it. fixed #325
2014-04-16 12:00:29 +08:00
fengmk2
a04f5d68eb if download tarball 404, throw err better than ignore it. fixed #325 2014-04-16 11:44:28 +08:00
fengmk2
7cfe3b58ce refator sync 2014-04-16 11:19:32 +08:00
fengmk2
afdf0bc653 Merge pull request #323 from cnpm/hot-fix-status
hotfix, close #319
2014-04-16 09:39:15 +08:00
fengmk2
42055ac91e Merge pull request #324 from cnpm/hotfix-dist
hotfix, close #321
2014-04-16 09:31:48 +08:00
dead_horse
b93e0dab41 hotfix, close #321 2014-04-16 09:28:22 +08:00
dead_horse
01f2187830 hotfix, close #319 2014-04-16 09:23:59 +08:00
fengmk2
c2f49fcdd9 support custom web home page 2014-04-14 15:14:28 +08:00
dead_horse
3925ef6044 Merge pull request #316 from cnpm/first-full-sync
npm get short only can read from cnpm now
2014-04-14 14:04:59 +08:00
fengmk2
85494d2ba0 npm get short only can read from cnpm now 2014-04-14 13:57:04 +08:00
dead_horse
660ca394d6 Merge pull request #315 from cnpm/local-bindding
Local bindding and redis fix
2014-04-14 10:47:39 +08:00
fengmk2
ad7eeac00c if using reverted proxy like nginx, only binding on local host 2014-04-14 10:43:49 +08:00
fengmk2
f63b72891b fix redis detect logic 2014-04-14 10:27:46 +08:00
fengmk2
28d31b093d Release 0.4.1 2014-04-10 14:46:01 +08:00
fengmk2
466e14e92a Merge pull request #313 from cnpm/hotfix-sync-status
fix sync status code error
2014-04-10 07:46:57 +08:00
dead_horse
3646c2ce50 fix sync status code error 2014-04-10 02:39:02 +08:00
fengmk2
fa44097d8f Release 0.4.0 2014-04-09 21:05:37 +08:00
dead_horse
0c56294c47 Merge pull request #312 from cnpm/local-mysql-dev-env
Local mysql dev env
2014-04-09 20:53:12 +08:00
fengmk2
41552fd27c fix test cases to run on local machine 2014-04-09 19:37:37 +08:00
fengmk2
cb4d4f51dd add contribute guidelines 2014-04-09 09:58:23 +08:00
fengmk2
1b266d527c use local mysql for dev env. fix #308 2014-04-09 09:31:34 +08:00
fengmk2
9d660be259 Merge pull request #306 from cnpm/use-copy-to
use copy to
2014-04-01 17:37:36 +08:00
dead_horse
b041cc7361 use copy to 2014-04-01 16:19:49 +08:00
dead_horse
e420f6985e Merge pull request #305 from cnpm/fix-test
fix test case
2014-03-31 17:51:15 +08:00
fengmk2
60dc5cf5fa fix test case 2014-03-31 17:47:10 +08:00
fengmk2
54916f49e5 Merge pull request #303 from cnpm/compress-conditional
use koa-compress and koa-conditional-get
2014-03-31 14:52:39 +08:00
dead_horse
a3cafaa297 use koa-compress and koa-conditional-get 2014-03-31 14:21:13 +08:00
fengmk2
aa13a100d4 Merge pull request #302 from cnpm/typeerror-hotfix
maintainers is string, fix #301
2014-03-30 09:01:23 +08:00
fengmk2
3b29310826 maintainers is string, fix #301 2014-03-30 09:01:03 +08:00
fengmk2
6d76a590e5 Release 0.3.13 2014-03-27 16:28:01 +08:00
fengmk2
3a98b63736 Merge pull request #299 from cnpm/fix-user-update
fix npm adduser update 409 bug
2014-03-27 16:24:34 +08:00
fengmk2
f5a2090fda Merge pull request #298 from cnpm/fix-ml-cov
fix multiline coverage
2014-03-27 16:18:23 +08:00
fengmk2
989a69143f fix npm adduser update 409 bug 2014-03-27 16:18:01 +08:00
dead_horse
77e6db4f0e fix multiline coverage 2014-03-27 16:03:00 +08:00
fengmk2
a3f7affe7b Release 0.3.13 2014-03-27 15:47:19 +08:00
dead_horse
2cd0453c6b Merge pull request #297 from cnpm/pkg-engine
show package engines. fixed #280
2014-03-27 15:23:32 +08:00
fengmk2
84bc126d57 show package engines. fixed #280 2014-03-27 15:13:18 +08:00
dead_horse
760421661c Merge pull request #296 from cnpm/sync-with-local
dont sync local package field. fix #295
2014-03-27 15:04:25 +08:00
fengmk2
425e430a3a dont sync local package field. fix #295 2014-03-27 14:55:52 +08:00
fengmk2
ad3602bf15 Release 0.3.12 2014-03-26 15:28:05 +08:00
fengmk2
2c5852b344 Merge pull request #294 from cnpm/hotfix-sync
fix result.successes not exist error
2014-03-26 15:22:59 +08:00
dead_horse
115a349fc5 fix result.successes not exist error 2014-03-26 15:21:37 +08:00
fengmk2
3ceea3ccf4 Merge pull request #293 from cnpm/search-list
fix search list
2014-03-24 14:20:51 +08:00
dead_horse
ab6a9b2bea fix search list 2014-03-24 14:17:22 +08:00
fengmk2
9cc84abbb5 Merge pull request #292 from cnpm/issue291-all
only return package name in /-/all and /-/all/since, fixed #291
2014-03-24 11:27:31 +08:00
dead_horse
c1c07cf2e4 add simple request for listall 2014-03-24 11:11:54 +08:00
dead_horse
ce5c97956b only return package name in /-/all and /-/all/since, fixed #291 2014-03-24 10:31:28 +08:00
dead_horse
7cb7a517da refine docs foloder 2014-03-23 02:28:03 +08:00
dead_horse
c7df6f0f1a fix typo 2014-03-23 02:26:38 +08:00
fengmk2
ee59176b67 Merge pull request #290 from cnpm/modified-as-etag
use module gmt_modified as etag. fix #288
2014-03-22 02:09:42 +08:00
fengmk2
f160d741cb use module gmt_modified as etag. fix #288 2014-03-22 01:55:13 +08:00
dead_horse
94974f81ae fix typo, remove unused config in package.json 2014-03-21 17:26:54 +08:00
fengmk2
41c0fea3dd Merge pull request #289 from cnpm/readme
Readme
2014-03-21 16:58:27 +08:00
dead_horse
dfae519e96 fix typo 2014-03-21 16:56:15 +08:00
dead_horse
4d7d4f06b1 web page only list cnpm registry related info 2014-03-21 16:25:47 +08:00
dead_horse
5a890ff27e update readme 2014-03-21 16:15:38 +08:00
fengmk2
ac124d0e2d Merge pull request #286 from cnpm/qnfs-generator
use generator in qnfs
2014-03-20 21:17:03 +08:00
dead_horse
7e267156eb use generator in qnfs 2014-03-20 20:35:58 +08:00
98 changed files with 4065 additions and 1366 deletions

4
.gitignore vendored
View File

@@ -7,6 +7,7 @@ coverage.html
*.pid
*.gz
dump.rdb
.DS_Store
pids
logs
@@ -15,11 +16,14 @@ results
node_modules
npm-debug.log
public/dist/
.dist
config/config.js
backup/*.json
backup/*.gz
docs/web/history.md
docs/web/_readme.md
view/web/_layout.html
bin/mysql.js
bin/test.sql
coverage/
config/web_readme.md

View File

@@ -58,7 +58,7 @@
"shadow" : true, // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
"sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation
"supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;`
"validthis" : false, // true: Tolerate using this in a non-constructor function
"validthis" : true, // true: Tolerate using this in a non-constructor function
// Environments
"browser" : true, // Web Browser (window, document, etc)

View File

@@ -8,9 +8,14 @@ logo.png
public/dist/
backup/*.json
backup/*.gz
docs/web/history.md
docs/web/_readme.md
view/web/_layout.html
bin/mysql.js
bin/test.sql
coverage/
.jshintrc
.jshintignore
.DS_Store
config/web_readme.md
.dist/

39
CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,39 @@
# How to contribute
Third-party patches are essential for keeping `cnpmjs.org` great.
We want to keep it as easy as possible to contribute changes that
get things working in your environment. There are a few guidelines that we
need contributors to follow so that we can have a chance of keeping on
top of things.
## Getting Started
* Make sure you have a [GitHub account](https://github.com/signup/free)
* Fork the repository on GitHub
## Making Changes
* Create a topic branch from where you want to base your work.
* This is usually the master branch.
* Only target release branches if you are certain your fix must be on that
branch.
* To quickly create a topic branch based on master.
Please avoid working directly on the `master` branch.
* Make commits of logical units and including unit tests.
* Check for unnecessary whitespace with `git diff --check` before committing.
* Make sure your commit messages are in the proper format.
* Make sure you have added the necessary tests for your changes.
* Run _all_ the tests to assure nothing else was accidentally broken.
* Follow [node style guide](https://github.com/felixge/node-style-guide)
## Submitting Changes
* Push your changes to a topic branch in your fork of the repository.
* Submit a pull request.
* Make sure travis-ci test pass.
# Additional Resources
* [General GitHub documentation](http://help.github.com/)
* [GitHub pull request documentation](http://help.github.com/send-pull-requests/)
* [cnpmjs.org](http://cnpmjs.org)

View File

@@ -1,44 +1,181 @@
0.3.11 / 2014-03-20
0.8.0 / 2014-07-21
==================
* use common.isMaintainer, fixed #283
* support "scoped" packages. close #352
* use safe jsonp
* Stop support old publish flow. fix #368
* update SQLs
* use sync_info and sync_error categories
* add categories to loggers. fix #370
* fix get latest tag always not exists bug
* support `npm publish --tag beta`. fix #366
* use mini-logger and error-formater
0.7.0 / 2014-07-07
==================
* use module_maintainers on GET /pakcage/:name page
* use new module_maintainers on GET /:name
* admin user should never publish to other user's packages. fix #363
* Add a new table for module-maintainers.
* gravatar use https
* support https
0.6.1 / 2014-06-18
==================
* hot fix removeTagsByNames()
* fix _rev not exists
* sync unpublished on GET /sync/:name
0.6.0 / 2014-06-16
==================
* sync unpublished info. close #353
* Delete not exists versions on sync worker. #353
0.5.3 / 2014-06-13
==================
* fix sync response 204
* add links in History.md
* bump koa
* fix test-cov
* bump koa and should
0.5.2 / 2014-06-04
==================
* sync hotfix
* sync phantomjs downloads pkg. close [#348](https://github.com/cnpm/cnpmjs.org/issues/348)
* add restart, fixed [#346](https://github.com/cnpm/cnpmjs.org/issues/346)
0.5.1 / 2014-05-28
==================
* fix attack on /-/all/since?stale=update_after&startkey=2 close [#336](https://github.com/cnpm/cnpmjs.org/issues/336)
* bump thunkify-wrap
* bump koa-middlewares
* remove outputError
* bump dependencies
* use svg badge
* add package/notfound page
* add dist mirror link to home page
* fix sync listdiff and add more test cases
0.5.0 / 2014-05-13
==================
* filter /nightlies/*
* use koa setter instead of set()
* add more info on error email
* add sync dist to sync/index.js
* show dist page
* sync dist file and save it to database
* disable gzip before [#335](https://github.com/cnpm/cnpmjs.org/issues/335) has fix
0.4.3 / 2014-04-18
==================
* Merge pull request [#334](https://github.com/cnpm/cnpmjs.org/issues/334) from cnpm/fix-permission
* add permission check to /:name/:tag
* Merge pull request [#333](https://github.com/cnpm/cnpmjs.org/issues/333) from cnpm/issue332-tag
* fix space
* add put /:name/:tag, close [#332](https://github.com/cnpm/cnpmjs.org/issues/332)
0.4.2 / 2014-04-17
==================
* sync interval config
* fix fav ico and show pkg size on pkg info page. fix [#318](https://github.com/cnpm/cnpmjs.org/issues/318)
* sync work sync one done must wait for a defer.setImmediate. fix [#328](https://github.com/cnpm/cnpmjs.org/issues/328)
* bump dep versions
* if download tarball 404, throw err better than ignore it. fixed [#325](https://github.com/cnpm/cnpmjs.org/issues/325)
* refator sync
* hotfix, close [#321](https://github.com/cnpm/cnpmjs.org/issues/321)
* hotfix, close [#319](https://github.com/cnpm/cnpmjs.org/issues/319)
* support custom web home page
* npm get short only can read from cnpm now
* if using reverted proxy like nginx, only binding on local host
* fix redis detect logic
0.4.1 / 2014-04-10
==================
* fix sync status code error
0.4.0 / 2014-04-09
==================
* fix test cases to run on local machine
* add contribute guidelines
* use local mysql for dev env. fix [#308](https://github.com/cnpm/cnpmjs.org/issues/308)
* use copy to
* use koa-compress and koa-conditional-get
* maintainers is string, fix [#301](https://github.com/cnpm/cnpmjs.org/issues/301)
0.3.13 / 2014-03-27
==================
* fix npm adduser update 409 bug
* fix multiline coverage
* show package engines. fixed [#280](https://github.com/cnpm/cnpmjs.org/issues/280)
* dont sync local package field. fix [#295](https://github.com/cnpm/cnpmjs.org/issues/295)
0.3.12 / 2014-03-26
==================
* fix result.successes not exist error
* fix search list
* add simple request for listall
* only return package name in /-/all and /-/all/since, fixed [#291](https://github.com/cnpm/cnpmjs.org/issues/291)
* refine docs foloder
* use module gmt_modified as etag. fix [#288](https://github.com/cnpm/cnpmjs.org/issues/288)
* fix typo, remove unused config in package.json
* web page only list cnpm registry related info
* use generator in qnfs
0.3.11 / 2014-03-20
==================
* use common.isMaintainer, fixed [#283](https://github.com/cnpm/cnpmjs.org/issues/283)
* update dependencies
* use co-mocha for test, fixed #279
* use co-mocha for test, fixed [#279](https://github.com/cnpm/cnpmjs.org/issues/279)
* update thunkify-wrap, breaking change in thunkify-wrap
* refactor SQLs by using multiline
* use multiline to refactor sqls
* ignore contributors
0.3.10 / 2014-03-16
0.3.10 / 2014-03-16
==================
* Only /_session request send the authSession. fixed #223
* sync npm user info when maintainers and contributors not exists. fixed #82
* Only /_session request send the authSession. fixed [#223](https://github.com/cnpm/cnpmjs.org/issues/223)
* sync npm user info when maintainers and contributors not exists. fixed [#82](https://github.com/cnpm/cnpmjs.org/issues/82)
* save npm user to mysql
* password salt always be randoms
* remove session access in /name and /name/version, fixed #274
* remove session access in /name and /name/version, fixed [#274](https://github.com/cnpm/cnpmjs.org/issues/274)
* fix update maintainer session error
* update koa-middlewares
* fix test, fix sync_by_install
* use defer session
* Support npm owner|author add [name] [pkg]. fixed #271
* Support npm owner|author add [name] [pkg]. fixed [#271](https://github.com/cnpm/cnpmjs.org/issues/271)
0.3.9 / 2014-03-14
0.3.9 / 2014-03-14
==================
* custom user-agent
* use co-urllib instead of thunkify urllib; fix mock http.request test cases
* request limit custom message
* add config.redis check
* add koa-limit, fixed #267
* add koa-limit, fixed [#267](https://github.com/cnpm/cnpmjs.org/issues/267)
0.3.8 / 2014-03-11
0.3.8 / 2014-03-11
==================
* update middlewares, fixed missing charset bug #264
* update middlewares, fixed missing charset bug [#264](https://github.com/cnpm/cnpmjs.org/issues/264)
0.3.7 / 2014-03-11
0.3.7 / 2014-03-11
==================
* show worker die date time
@@ -48,63 +185,63 @@
* fix return versions
* fix makefile, remove eventproxy
* refactor sync_module_worker
* add make test-dev, fixed #259
* add make test-dev, fixed [#259](https://github.com/cnpm/cnpmjs.org/issues/259)
* change npm.js to generator
* update urllib, proxy/npm.js use generator
* sync_all and sync_exist to generator
* change function to generator
* need node >= v0.11.9
0.3.6 / 2014-03-06
0.3.6 / 2014-03-06
==================
* install missing package should sync it from source npm. fixed #252
* install missing package should sync it from source npm. fixed [#252](https://github.com/cnpm/cnpmjs.org/issues/252)
* npm publish dont contains .jshint*
* npm test run jshint
* Add jshint check: $ make jshint
* use `yield* next` instead of `yield next`
* replace dist.u.qiniudn.com with cnpmjs.org/dist
0.3.5 / 2014-03-05
0.3.5 / 2014-03-05
==================
* redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249
* redirect /name to /package/name when /name is 404. fixed #245
* Add missing properies and sync missing star users. fixed #235
* redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed [#249](https://github.com/cnpm/cnpmjs.org/issues/249)
* redirect /name to /package/name when /name is 404. fixed [#245](https://github.com/cnpm/cnpmjs.org/issues/245)
* Add missing properies and sync missing star users. fixed [#235](https://github.com/cnpm/cnpmjs.org/issues/235)
0.3.4 / 2014-03-04
0.3.4 / 2014-03-04
==================
* add cov
* use istanbul run test coverage
* gzip support. fix #241
* gzip support. fix [#241](https://github.com/cnpm/cnpmjs.org/issues/241)
* readme spelling patch (@stanzheng)
* default readme to null, fixed #233
* default readme to null, fixed [#233](https://github.com/cnpm/cnpmjs.org/issues/233)
* remove readme in versions
0.3.3 / 2014-02-28
0.3.3 / 2014-02-28
==================
* Merge pull request #232 from cnpm/host-hotfix
* Merge pull request [#232](https://github.com/cnpm/cnpmjs.org/issues/232) from cnpm/host-hotfix
* get request host from request.headers
* Merge pull request #231 from cnpm/bug-fix
* fix deps display bug#230 and nsf.url TypeError#229
* Merge pull request [#231](https://github.com/cnpm/cnpmjs.org/issues/231) from cnpm/bug-fix
* fix deps display bug[#230](https://github.com/cnpm/cnpmjs.org/issues/230) and nsf.url TypeError[#229](https://github.com/cnpm/cnpmjs.org/issues/229)
0.3.2 / 2014-02-28
0.3.2 / 2014-02-28
==================
* update koa-sess and koa-redis
* fix sync all test
* remove nfs.downloadStream first, fix tmppath error
* fix fengmk2/giturl#1 bug
* fix fengmk2/giturl[#1](https://github.com/cnpm/cnpmjs.org/issues/1) bug
0.3.1 / 2014-02-27
0.3.1 / 2014-02-27
==================
* add etag fixed #224
* add etag fixed [#224](https://github.com/cnpm/cnpmjs.org/issues/224)
* travis ci install on source npm
0.3.0 / 2014-02-27
0.3.0 / 2014-02-27
==================
* fix typo and dont sync not exists pkgs
@@ -125,162 +262,162 @@
* fix all the test of registry module.test.js
* convert registry/module.js to koa type
* fix auth middleware
* finish registry user controller koa and update mm to support thunkify. fixed #196
* finish registry user controller koa and update mm to support thunkify. fixed [#196](https://github.com/cnpm/cnpmjs.org/issues/196)
* change controllers/user.js to koa
* thunkify all proxy
* convert all middlewares to koa type
* change regsitry sync to koa
* addd koa-jsonp, koa-bodyparser, fix / controller
* first koa run registry home page /
* Merge pull request #212 from cnpm/fix-sync-404
* Merge pull request [#212](https://github.com/cnpm/cnpmjs.org/issues/212) from cnpm/fix-sync-404
* return friendly 404 reason
* Merge pull request #211 from cnpm/bug-fix
* override json limit to default 10mb. fixed #209
* fix #210 addPackageAndDist package version detect bug
* Merge pull request [#211](https://github.com/cnpm/cnpmjs.org/issues/211) from cnpm/bug-fix
* override json limit to default 10mb. fixed [#209](https://github.com/cnpm/cnpmjs.org/issues/209)
* fix [#210](https://github.com/cnpm/cnpmjs.org/issues/210) addPackageAndDist package version detect bug
0.2.27 / 2014-02-19
0.2.27 / 2014-02-19
==================
* support json result in search, fixed #189
* support json result in search, fixed [#189](https://github.com/cnpm/cnpmjs.org/issues/189)
0.2.26 / 2014-02-19
0.2.26 / 2014-02-19
==================
* npm publish also need to add deps
0.2.25 / 2014-02-19
0.2.25 / 2014-02-19
==================
* max handle number of package.json `dependencies` property
* Dependents support. fixed #190
* Dependents support. fixed [#190](https://github.com/cnpm/cnpmjs.org/issues/190)
0.2.24 / 2014-02-13
0.2.24 / 2014-02-13
==================
* fix if delete all the versions
* refactor remove module, fixed #186
* refactor remove module, fixed [#186](https://github.com/cnpm/cnpmjs.org/issues/186)
0.2.23 / 2014-01-26
0.2.23 / 2014-01-26
==================
* system admin can add, publish, remove the packages. fixed #176
* system admin can add, publish, remove the packages. fixed [#176](https://github.com/cnpm/cnpmjs.org/issues/176)
0.2.22 / 2014-01-26
0.2.22 / 2014-01-26
==================
* add keyword and search support keyword. #181
* add keyword and search support keyword. [#181](https://github.com/cnpm/cnpmjs.org/issues/181)
0.2.21 / 2014-01-24
0.2.21 / 2014-01-24
==================
* refactor code styles on package.html
* nav-tabs e.preventDefault
* Show registry server error response. fixed #178
* Show registry server error response. fixed [#178](https://github.com/cnpm/cnpmjs.org/issues/178)
* nav-tabs for package.html (@4simple)
0.2.20 / 2014-01-23
0.2.20 / 2014-01-23
==================
* hotfix sync missing dependencies and readmes
* fix sync readme error, fixed #174
* fix sync readme error, fixed [#174](https://github.com/cnpm/cnpmjs.org/issues/174)
* add updateReadme in module
0.2.19 / 2014-01-22
0.2.19 / 2014-01-22
==================
* npm install no need to check authorization header. fixed #171
* npm install no need to check authorization header. fixed [#171](https://github.com/cnpm/cnpmjs.org/issues/171)
0.2.18 / 2014-01-20
0.2.18 / 2014-01-20
==================
* Support gitlab git url to display and click. fixed #160
* Support gitlab git url to display and click. fixed [#160](https://github.com/cnpm/cnpmjs.org/issues/160)
* fix redis crash
0.2.17 / 2014-01-17
0.2.17 / 2014-01-17
==================
* custom logo url
* hotfix layout bug
0.2.16 / 2014-01-16
0.2.16 / 2014-01-16
==================
* fix publish-time bug
0.2.15 / 2014-01-16
0.2.15 / 2014-01-16
==================
* add publish_time to debug
0.2.14 / 2014-01-16
0.2.14 / 2014-01-16
==================
* add make autod
* update publish_time, fixed #163
* update publish_time, fixed [#163](https://github.com/cnpm/cnpmjs.org/issues/163)
0.2.13 / 2014-01-15
0.2.13 / 2014-01-15
==================
* markdown tmpl not support footer, need to wrap on app start
0.2.12 / 2014-01-15
0.2.12 / 2014-01-15
==================
* add footer and npm client name customable
0.2.11 / 2014-01-15
0.2.11 / 2014-01-15
==================
* package page contributor link to search, default is true
0.2.10 / 2014-01-14
0.2.10 / 2014-01-14
==================
* fix #155 Content-Disposition wrong.
* fix [#155](https://github.com/cnpm/cnpmjs.org/issues/155) Content-Disposition wrong.
0.2.9 / 2014-01-14
0.2.9 / 2014-01-14
==================
* support startkey=c and startkey="c"
* support couch db search api. fixed #153
* support couch db search api. fixed [#153](https://github.com/cnpm/cnpmjs.org/issues/153)
* fix fork me image link
* support sync by query.name
0.2.8 / 2014-01-14
0.2.8 / 2014-01-14
==================
* dont show err stack on test env
* add download link for package page
0.2.7 / 2014-01-13
0.2.7 / 2014-01-13
==================
* add shasum when nfs.upload and hfs.uploadBuffer, fixed #148
* add shasum when nfs.upload and hfs.uploadBuffer, fixed [#148](https://github.com/cnpm/cnpmjs.org/issues/148)
0.2.6 / 2014-01-13
0.2.6 / 2014-01-13
==================
* support custom session store, fixed #146
* support custom session store, fixed [#146](https://github.com/cnpm/cnpmjs.org/issues/146)
0.2.5 / 2014-01-13
0.2.5 / 2014-01-13
==================
* add download timeout and unit test
* use downloadStream() first
* nfs download to a writeable stream.
0.2.4 / 2014-01-10
0.2.4 / 2014-01-10
==================
* set main script to index.js, fixed #142
* set main script to index.js, fixed [#142](https://github.com/cnpm/cnpmjs.org/issues/142)
0.2.3 / 2014-01-10
0.2.3 / 2014-01-10
==================
* Dont show sync button on private package
* Sync package as publish with no deps. fixed #138
* Sync package as publish with no deps. fixed [#138](https://github.com/cnpm/cnpmjs.org/issues/138)
0.2.2 / 2014-01-10
0.2.2 / 2014-01-10
==================
* keep compatibility
@@ -290,23 +427,23 @@
* new npm publish in one request, add _publish_in_cnpm
* support unsure name ufs
* contributors maybe a object
* Object #<Object> has no method 'forEach' fixed #134
* support custom config as a module, fixed issue #132
* support npm new publish flow. fixed #129
* Object #<Object> has no method 'forEach' fixed [#134](https://github.com/cnpm/cnpmjs.org/issues/134)
* support custom config as a module, fixed issue [#132](https://github.com/cnpm/cnpmjs.org/issues/132)
* support npm new publish flow. fixed [#129](https://github.com/cnpm/cnpmjs.org/issues/129)
* add toString and constructor to test admin
* fix #119 hasOwnProperty check admin bug.
* fix [#119](https://github.com/cnpm/cnpmjs.org/issues/119) hasOwnProperty check admin bug.
0.2.0 / 2013-12-27
0.2.0 / 2013-12-27
==================
* remove to lower case
* fix #127 execSync and execsync.
* fix [#127](https://github.com/cnpm/cnpmjs.org/issues/127) execSync and execsync.
* add contributors list on package page
* mv blanket to config
* sync typeerror fix #statusCode
* add disturl
* fix #122 admin security bug
* fixed #121, let pkg 404 as success
* fix [#122](https://github.com/cnpm/cnpmjs.org/issues/122) admin security bug
* fixed [#121](https://github.com/cnpm/cnpmjs.org/issues/121), let pkg 404 as success
* fix sql insert error
* fix typos
@@ -318,78 +455,78 @@
* make sure all packages name are lower case
* select ids from tag
* fix nodejsctl
* fix #112 missing versions and time no sync
* fix [#112](https://github.com/cnpm/cnpmjs.org/issues/112) missing versions and time no sync
* remove restart command
* fix sync missing packages error
* fix web/readme.md, add install
* fix #109 pkg no times and no versions bug.
* fix [#109](https://github.com/cnpm/cnpmjs.org/issues/109) pkg no times and no versions bug.
0.1.2 / 2013-12-19
==================
* fix times not exists canot sync bug. fixed #101
* fix times not exists canot sync bug. fixed [#101](https://github.com/cnpm/cnpmjs.org/issues/101)
* support npm run command
* remove before_install and install in travis, fixed #102
* split all sub queries, fixed #104
* fix doc, fixed #103
* remove before_install and install in travis, fixed [#102](https://github.com/cnpm/cnpmjs.org/issues/102)
* split all sub queries, fixed [#104](https://github.com/cnpm/cnpmjs.org/issues/104)
* fix doc, fixed [#103](https://github.com/cnpm/cnpmjs.org/issues/103)
* fix search too slow.
* dont email sync log level info
* only sync missing packages at first time
* update dependencies
* sync all will sync all the missing packages, fixed #97
* sync all will sync all the missing packages, fixed [#97](https://github.com/cnpm/cnpmjs.org/issues/97)
0.1.0 / 2013-12-12
==================
* add sync title
* add favicon. fixed #69
* refine sync page, fiexd #70
* add favicon. fixed [#69](https://github.com/cnpm/cnpmjs.org/issues/69)
* refine sync page, fiexd [#70](https://github.com/cnpm/cnpmjs.org/issues/70)
* add app version
* add test for sync
* refine sync page
* registry and web all use controllers/sync.js
* sync from web, fixed #58
* sync from web, fixed [#58](https://github.com/cnpm/cnpmjs.org/issues/58)
* saving missing descriptions
* add package download info. fixed #63
* add package download info. fixed [#63](https://github.com/cnpm/cnpmjs.org/issues/63)
* add avatar
* use dependecies, fixed #issue62
* support open search, fixed #60
* make sure publish_time and author is same to source npm registry. fixed #56
* support open search, fixed [#60](https://github.com/cnpm/cnpmjs.org/issues/60)
* make sure publish_time and author is same to source npm registry. fixed [#56](https://github.com/cnpm/cnpmjs.org/issues/56)
* add test for search
* add a simple search by mysql like
* fix This version of MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery. fixed #54
* fix This version of MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery. fixed [#54](https://github.com/cnpm/cnpmjs.org/issues/54)
* update install doc, use nodejsctl to start
* must add limit on list by author sql
* fix sql, change test to fit my local database, fixed #46
* fix sql, change test to fit my local database, fixed [#46](https://github.com/cnpm/cnpmjs.org/issues/46)
* use registry.cnpmjs.org
* add install document and total package info on home page. fix #42
* add module_id to tag table. #46
* skip error version. fixed #43
* add install document and total package info on home page. fix [#42](https://github.com/cnpm/cnpmjs.org/issues/42)
* add module_id to tag table. [#46](https://github.com/cnpm/cnpmjs.org/issues/46)
* skip error version. fixed [#43](https://github.com/cnpm/cnpmjs.org/issues/43)
* sync may make a user do not exist in database, but have modules in registry
* add user page
* fix set license
* ignore 404 on sync. fixed #39
* ignore 404 on sync. fixed [#39](https://github.com/cnpm/cnpmjs.org/issues/39)
* fix module page, add test
* update urllib to 0.5.5
* version and tag
* add module page
* fix download url
* first get tag, then try version
* support sync triggle by install, finish #31
* support sync triggle by install, finish [#31](https://github.com/cnpm/cnpmjs.org/issues/31)
* addTag error return 500
* just one download field
* add download total info on home page
* add download count
* versions empty and also check missing tags
* remove tags on unpublish
* add module tag. fix #6
* add module tag. fix [#6](https://github.com/cnpm/cnpmjs.org/issues/6)
* add [done] flag to check sync done on client
* get sync log #29
* get sync log [#29](https://github.com/cnpm/cnpmjs.org/issues/29)
* fix test in module
* rm tmp file on down request error
* add time for debug str
* fix pkg not exists null bug
* use sync module woker to handle sync process. fixed #19
* use sync module woker to handle sync process. fixed [#19](https://github.com/cnpm/cnpmjs.org/issues/19)
* if private mode enable, only admin can publish module
* add alias in readme
* fix sql, add sort by name
@@ -398,15 +535,15 @@
* add npm and cnpm image
* add registry total info on home page
* fix mods bug in module.removeAll, change module.update => module.removeWithVersions
* add test, fix bug. fixed #18
* add test, fix bug. fixed [#18](https://github.com/cnpm/cnpmjs.org/issues/18)
* spoort unpublish
* add web page index readme
* switchable nfs #21
* switchable nfs [#21](https://github.com/cnpm/cnpmjs.org/issues/21)
* change file path to match npm file path
* use qn cdn to store tarball file fixed #16
* add GET /:name/:version, fixed #3
* use qn cdn to store tarball file fixed [#16](https://github.com/cnpm/cnpmjs.org/issues/16)
* add GET /:name/:version, fixed [#3](https://github.com/cnpm/cnpmjs.org/issues/3)
* add module controller test cases; fix next module not exists logic bug.
* publish module flow finish #11
* publish module flow finish [#11](https://github.com/cnpm/cnpmjs.org/issues/11)
* add test for controllers/registry/user.js
* add test for middleware/auth
* add test for proxy/user
@@ -417,9 +554,9 @@
* add start time
* add home page
* remove session controller
* adduser() finish fixed #5
* adduser() finish fixed [#5](https://github.com/cnpm/cnpmjs.org/issues/5)
* rm app.js and routes.js
* Mock npm adduser server response, fixing #5
* Mock npm adduser server response, fixing [#5](https://github.com/cnpm/cnpmjs.org/issues/5)
* adjust project dir, separate registry and web server
* Init rest frame for cnpmjs.org
* init

View File

@@ -1,52 +1,53 @@
TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
REPORTER = tap
REPORTER = spec
TIMEOUT = 30000
MOCHA_OPTS =
REGISTRY = --registry=https://registry.npm.taobao.org
install:
@npm install --registry=http://r.cnpmjs.org \
--disturl=http://dist.cnpmjs.org
@npm install $(REGISTRY) \
--disturl=https://npm.taobao.org/dist
jshint:
jshint: install
@-./node_modules/.bin/jshint ./
test:
pretest:
@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
@mysql -uroot 'cnpmjs_test' < ./docs/db.sql
@mysql -uroot 'cnpmjs_test' -e 'show tables;'
test: install pretest
@NODE_ENV=test ./node_modules/.bin/mocha \
--harmony-generators \
--harmony \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
--require should \
--require co-mocha\
--require should-http \
--require co-mocha \
--require ./test/init.js \
$(MOCHA_OPTS) \
$(TESTS)
test-cov:
test-cov cov: install pretest
@NODE_ENV=test node --harmony \
node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha \
node_modules/.bin/istanbul cover --preserve-comments \
./node_modules/.bin/_mocha \
-- -u exports \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
--require should \
--require co-mocha\
--require should-http \
--require co-mocha \
--require ./test/init.js \
$(MOCHA_OPTS) \
$(TESTS)
@-$(MAKE) check-coverage
check-coverage:
@./node_modules/.bin/istanbul check-coverage \
--statements 100 \
--functions 100 \
--branches 100 \
--lines 100
cov:
@./node_modules/.bin/cov coverage
contributors:
contributors: install
@./node_modules/.bin/contributors -f plain -o AUTHORS
autod:
autod: install
@./node_modules/.bin/autod -w -e public,view,docs,backup,coverage
@$(MAKE) install

View File

@@ -1,40 +1,95 @@
cnpmjs.org
=======
[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.svg)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.svg)](https://gemnasium.com/cnpm/cnpmjs.org)
[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
[![NPM](https://nodei.co/npm/cnpmjs.org.svg?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
## What is this?
Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
Private npm registry and web for Enterprise, base on [koa](http://koajs.com/),
MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
Our goal is to provide a low cost maintenance and easy to use solution for private npm.
## What can you do with `cnpmjs.org`
* Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now)
* Build a mirror NPM. (we use it to build a mirror in China: [cnpmjs.org](http://cnpmjs.org/))
* Build a completely independent NPM registry to store whatever you like.
### Features
* **Support "scoped" packages**: [npm/npm#5239](https://github.com/npm/npm/issues/5239)
* **Simple to deploy**: only need `mysql` and a [simple store system](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
You can get the source code through `npm` or `git`.
* **Low cost and easy maintenance**: `package.json` info store in MySQL, tarball(tgz file) store in CDN or other store systems.
* **Automatic synchronization**: automatic synchronization from any registry specified, support two sync modes:
- Sync all modules from a specified registry, like [npm registry](http://registry.npmjs.org).
- Only sync the modules that exists in your own registry.
* **Manual synchronization**: automatic synchronization may has little delay, but you can syn immediately by manually.
* **Customized client**: we provide a client [cnpm](https://github.com/cnpm/cnpm)
to extend `npm` with more features(`sync` command, [gzip](https://github.com/npm/npm-registry-client/pull/40) support).
And it easy to wrap for your own registry which build with `cnpmjs.org`.
* **Compatible with NPM client**: you can use the origin NPM client with `cnpmjs.org`,
only need to change the registry in config. Even include manual synchronization (through `install` command).
## Getting Start
* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
* @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
* install and deploy cnpmjs.org through npm: [examples](https://github.com/cnpm/custom-cnpm-example)
* Mirror NPM in China: [cnpmjs.org](http://cnpmjs.org)
* cnpm client: [cnpm](https://github.com/cnpm/cnpm), `npm install -g cnpm`
* [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
* [NFS guide](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide)
![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
## Develop on your local machine
## Install
### Dependencies
* [node](http://nodejs.org) =0.11.12
* [mysql](http://dev.mysql.com/downloads/) >= 0.5.0, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
### Start MySQL
```bash
$ npm install --registry=http://r.cnpmjs.org --disturl=http://cnpmjs.org/dist
$ nohup mysqld &
```
## Usage
### Clone codes and run test
```js
$ node --harmony-generators dispatch.js
```bash
# clone from git
$ git clone https://github.com/cnpm/cnpmjs.org.git
# install dependencies
$ make install
# test
$ make test
# coverage
$ make test-cov
# udpate dependencies
$ make autod
# start server
$ node --harmony_generators dispatch.js
```
**Notice**: need node version >=0.11.9
## How to contribute
## Guide
* Clone the project
* Checkout a new branch
* Add new features or fix bugs in the new branch
* Make a pull request and we will review it ASAP
* [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
* [NFS guide](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide)
Tips: make sure your code is following the [node-style-guide](https://github.com/felixge/node-style-guide).
## Authors
@@ -42,15 +97,16 @@ $ node --harmony-generators dispatch.js
$ git summary
project : cnpmjs.org
repo age : 3 months
active : 145 days
commits : 366
files : 94
repo age : 4 months ago
commits : 472
active : 167 days
files : 104
authors :
217 fengmk2 59.3%
146 dead_horse 39.9%
2 4simple 0.5%
1 Alsotang 0.3%
272 fengmk2 57.6%
195 dead_horse 41.3%
2 4simple 0.4%
2 Stanley Zheng 0.4%
1 Alsotang 0.2%
```
## License

View File

@@ -7,7 +7,7 @@ export NODE_ENV='production'
ulimit -c unlimited
cd `dirname $0`/..
NODEJS='node --harmony-generators'
NODEJS='node --harmony'
BASE_HOME=`pwd`
PROJECT_NAME=`basename ${BASE_HOME}`
STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\

View File

@@ -15,73 +15,48 @@
* Module dependencies.
*/
var util = require('util');
var moment = require('moment');
var logstream = require('logfilestream');
var ms = require('ms');
var formater = require('error-formater');
var Logger = require('mini-logger');
var config = require('../config');
var utility = require('utility');
var mail = require('./mail');
var util = require('util');
var isTEST = process.env.NODE_ENV === 'test';
var ONE_DAY = ms('1d');
var levels = ['info', 'warn', 'error'];
var categories = ['sync_info', 'sync_error'];
levels.forEach(function (catetory) {
var options = {
logdir: config.logdir,
duration: ONE_DAY,
nameformat: '[' + catetory + '.]YYYY-MM-DD[.log]'
};
var stream = logstream(options);
function write(msg) {
var time = moment().format('YYYY-MM-DD HH:mm:ss.SSS');
var subject = null;
if (msg instanceof Error) {
subject = msg.name;
var err = {
name: msg.name,
code: msg.code,
message: msg.message,
stack: msg.stack,
host: msg.host,
url: msg.url,
data: msg.data
};
if (err.name === 'Error' && typeof err.code === 'string') {
err.name = err.code + err.name;
}
err.name += 'Exception';
if (err.host) {
err.message += ' (' + err.host + ')';
}
msg = util.format('%s nodejs.%s: %s\nURL: %s\nData: %j\n%s\n\n',
time,
err.name,
err.stack,
err.url,
err.data,
time
);
} else {
msg = time + ' ' + util.format.apply(util, arguments) + '\n';
}
if (!isTEST) {
stream.write(msg);
if (config.debug) {
var level = catetory;
console.log('[' + level + '] ' + msg);
} else {
if (catetory === 'error' && subject) {
// send error email
var to = [];
for (var name in config.admins) {
to.push(config.admins[name]);
}
mail.error(to, subject, msg);
}
}
}
}
exports[catetory] = write;
var logger = module.exports = Logger({
categories: categories,
dir: config.logdir,
duration: '1d',
format: '[{category}.]YYYY-MM-DD[.log]',
stdout: config.debug && !isTEST,
errorFormater: errorFormater
});
var to = [];
for (var name in config.admins) {
to.push(config.admins[name]);
}
function errorFormater(err) {
var msg = formater.both(err);
mail.error(to, msg.json.name, msg.text);
return msg.text;
}
logger.syncInfo = function () {
var args = [].slice.call(arguments);
if (typeof args[0] === 'string') {
args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0];
}
logger.sync_info.apply(logger, args);
};
logger.syncError =function () {
var args = [].slice.call(arguments);
if (typeof args[0] === 'string') {
args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0];
}
logger.sync_error.apply(logger, arguments);
};

View File

@@ -16,9 +16,12 @@
var thunkify = require('thunkify-wrap');
var qn = require('qn');
var fs = require('fs');
var config = require('../config');
var client = qn.create(config.qn);
thunkify(client, ['delete', 'uploadFile', 'upload', 'download']);
exports._client = client;
/**
@@ -28,38 +31,53 @@ exports._client = client;
* @param {Object} options
* - {String} key
* - {Number} size
* @param {Function(err, result)} callback
* - {Object} result
* - {String} url
*/
exports.upload = function (filepath, options, callback) {
client.delete(options.key, function (err, data) {
client.uploadFile(filepath, {key: options.key, size: options.size}, function (err, data) {
if (err) {
return callback(err);
}
callback(null, {url: data.url});
});
exports.upload = function *(filepath, options) {
try {
yield client.delete(options.key);
} catch (err) {
// ignore error here
}
var res = yield client.uploadFile(filepath, {
key: options.key,
size: options.size
});
var url = res && res[0] ? res[0].url : '';
return { url: url };
};
exports.uploadBuffer = function (buf, options, callback) {
client.delete(options.key, function (err, data) {
client.upload(buf, {key: options.key}, function (err, data) {
if (err) {
return callback(err);
}
callback(null, {url: data.url});
});
});
exports.uploadBuffer = function *(buf, options) {
try {
yield client.delete(options.key);
} catch (err) {
// ignore error here
}
var res = yield client.upload(buf, {key: options.key});
var url = res && res[0] ? res[0].url : '';
return { url: url };
};
exports.url = function (key) {
return client.resourceURL(key);
};
exports.remove = function (key, callback) {
client.delete(key, callback);
exports.download = function* (key, filepath, options) {
var writeStream = fs.createWriteStream(filepath);
yield client.download(key, {
timeout: options.timeout,
writeStream: writeStream
});
};
thunkify(exports);
exports.remove = function *(key) {
try {
return yield client.delete(key);
} catch (err) {
if (err.name === 'QiniuFileNotExistsError') {
return;
}
throw err;
}
};

View File

@@ -17,7 +17,7 @@
var config = require('../config');
// close redis by set config.redis to `null` or `{}`
if (!config.redis || !config.redis.host || !config.redis.port) {
if (config.redis && config.redis.host && config.redis.port) {
var redis = require('redis');
var wrapper = require('co-redis');

View File

@@ -19,6 +19,7 @@ var path = require('path');
var fs = require('fs');
var os = require('os');
var mkdirp = require('mkdirp');
var copy = require('copy-to');
fs.existsSync = fs.existsSync || path.existsSync;
var version = require('../package.json').version;
@@ -29,6 +30,7 @@ var config = {
version: version,
registryPort: 7001,
webPort: 7002,
bindingHost: '127.0.0.1', // only binding on 127.0.0.1 for local access
enableCluster: false,
numCPUs: os.cpus().length,
debug: true, // if debug
@@ -37,30 +39,34 @@ var config = {
// mysql config
mysqlServers: [
{
host: 'keydiary.mysql.rds.aliyuncs.com', // 'db4free.net'
host: '127.0.0.1',
port: 3306,
user: 'cnpmjs',
password: 'cnpmjs123'
user: 'root',
password: ''
}
],
mysqlDatabase: 'cnpmjstest',
mysqlDatabase: 'cnpmjs_test',
mysqlMaxConnections: 4,
mysqlQueryTimeout: 5000,
sessionSecret: 'cnpmjs.org test session secret',
redis: {
host: 'pub-redis-19533.us-east-1-4.3.ec2.garantiadata.com',
port: 19533,
pass: 'cnpmjs_dev'
// host: 'pub-redis-19533.us-east-1-4.3.ec2.garantiadata.com',
// port: 19533,
// pass: 'cnpmjs_dev'
},
jsonLimit: '10mb', // max request json body size
uploadDir: path.join(root, 'public', 'dist'),
uploadDir: path.join(root, '.dist'),
// qiniu cdn: http://www.qiniu.com/, it free for dev.
qn: {
accessKey: "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
secretKey: "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j",
bucket: "qtestbucket",
domain: "http://qtestbucket.qiniudn.com"
// accessKey: "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
// secretKey: "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j",
// bucket: "qtestbucket",
// domain: "http://qtestbucket.qiniudn.com",
accessKey: "5UyUq-l6jsWqZMU6tuQ85Msehrs3Dr58G-mCZ9rE",
secretKey: "YaRsPKiYm4nGUt8mdz2QxeV5Q_yaUzVxagRuWTfM",
bucket: "qiniu-sdk-test",
domain: "http://qiniu-sdk-test.qiniudn.com",
},
mail: {
@@ -74,15 +80,20 @@ var config = {
debug: false
},
disturl: 'http://dist.u.qiniudn.com',
logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
noticeSyncDistError: true,
disturl: 'http://nodejs.org/dist',
syncDist: false,
logoURL: '//ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
registryHost: 'r.cnpmjs.org',
// customReadmeFile: __dirname + '/web_readme.md',
customReadmeFile: '', // you can use your custom readme file instead the cnpm one
customFooter: '', // you can add copyright and site total script html here
npmClientName: 'cnpm', // use `${name} install package`
packagePageContributorSearch: true, // package page contributor link to search, default is true
sourceNpmRegistry: 'http://registry.npmjs.org',
enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
admins: {
// name: email
fengmk2: 'fengmk2@gmail.com',
admin: 'admin@cnpmjs.org',
dead_horse: 'dead_horse@qq.com',
@@ -92,6 +103,7 @@ var config = {
backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
syncModel: 'none', // 'none', 'all', 'exist'
syncConcurrency: 1,
syncInterval: '10m', // sync interval, default is 10 minutes
maxDependencies: 200, // max handle number of package.json `dependencies` property
limit: {
@@ -102,16 +114,14 @@ var config = {
whiteList: [],
blackList: [],
message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
}
},
enableCompress: false, // enable gzip response or not
};
// load config/config.js, everything in config.js will cover the same key in index.js
var customConfig = path.join(root, 'config/config.js');
if (fs.existsSync(customConfig)) {
var options = require(customConfig);
for (var k in options) {
config[k] = options[k];
}
copy(require(customConfig)).override(config);
}
mkdirp.sync(config.logdir);
@@ -123,7 +133,5 @@ config.loadConfig = function (customConfig) {
if (!customConfig) {
return;
}
for (var key in customConfig) {
config[key] = customConfig[key];
}
copy(customConfig).override(config);
};

View File

@@ -18,6 +18,7 @@
var debug = require('debug')('cnpmjs.org:controllers:registry:module');
var path = require('path');
var fs = require('fs');
var util = require('util');
var crypto = require('crypto');
var utility = require('utility');
var coRead = require('co-read');
@@ -36,26 +37,82 @@ var SyncModuleWorker = require('../../proxy/sync_module_worker');
var logger = require('../../common/logger');
var ModuleDeps = require('../../proxy/module_deps');
var ModuleStar = require('../../proxy/module_star');
var ModuleUnpublished = require('../../proxy/module_unpublished');
var packageService = require('../../services/package');
var downloadAsReadStream = require('../utils').downloadAsReadStream;
/**
* show all version of a module
* GET /:name
*/
exports.show = function *(next) {
var name = this.params.name;
exports.show = function* (next) {
var name = this.params.name || this.params[0];
var rs = yield [
Module.getLastModified(name),
Module.listTags(name)
];
var modifiedTime = rs[0];
var tags = rs[1];
debug('show %s, last modified: %s, tags: %j', name, modifiedTime, tags);
if (modifiedTime) {
// find out the latest modfied time
// because update tags only modfied tag, wont change module gmt_modified
for (var i = 0; i < tags.length; i++) {
var tag = tags[i];
if (tag.gmt_modified > modifiedTime) {
modifiedTime = tag.gmt_modified;
}
}
// use modifiedTime as etag
this.set('ETag', '"' + modifiedTime.getTime() + '"');
// must set status first
this.status = 200;
if (this.fresh) {
debug('%s not change at %s, 304 return', name, modifiedTime);
this.status = 304;
return;
}
}
var r = yield [
Module.listTags(name),
Module.listByName(name),
ModuleStar.listUsers(name)
ModuleStar.listUsers(name),
packageService.listMaintainers(name),
];
var tags = r[0];
var rows = r[1];
var users = r[2];
var rows = r[0];
var users = r[1];
var maintainers = r[2];
debug('show %s got %d rows, %d tags, %d star users, maintainers: %j',
name, rows.length, tags.length, users.length, maintainers);
var userMap = {};
for (var i = 0; i < users.length; i++) {
userMap[users[i]] = true;
}
users = userMap;
if (rows.length === 0) {
// check if unpublished
var unpublishedInfo = yield* ModuleUnpublished.get(name);
debug('show unpublished %j', unpublishedInfo);
if (unpublishedInfo) {
this.status = 404;
this.body = {
_id: name,
name: name,
time: {
modified: unpublishedInfo.package.time,
unpublished: unpublishedInfo.package,
},
_attachments: {}
};
return;
}
}
// if module not exist in this registry,
// sync the module backend and return package info from official registry
if (rows.length === 0) {
@@ -68,8 +125,8 @@ exports.show = function *(next) {
return;
}
var result = yield SyncModuleWorker.sync(name, 'sync-by-install');
this.status = result.ok ? 200 : result.statusCode;
this.body = result.pkg;
this.status = result.ok ? 200 : (result.statusCode || 500);
return;
}
@@ -88,7 +145,6 @@ exports.show = function *(next) {
var times = {};
var attachments = {};
var createdTime = null;
var modifiedTime = null;
for (var i = 0; i < rows.length; i++) {
var row = rows[i];
if (row.version === 'next') {
@@ -105,13 +161,14 @@ exports.show = function *(next) {
readme = pkg.readme;
}
delete pkg.readme;
if (maintainers.length > 0) {
// TODO: need to use newer maintainers
pkg.maintainers = maintainers;
}
if (!createdTime || t < createdTime) {
createdTime = t;
}
if (!modifiedTime || t > modifiedTime) {
modifiedTime = t;
}
}
if (modifiedTime && createdTime) {
@@ -140,6 +197,12 @@ exports.show = function *(next) {
var pkg = latestMod.package;
if (tags.length === 0 && pkg.version !== 'next') {
// some sync error reason, will cause tags missing
// set latest tag at least
distTags.latest = pkg.version;
}
var info = {
_id: name,
_rev: rev,
@@ -162,24 +225,35 @@ exports.show = function *(next) {
info.license = pkg.license;
debug('show module %s: %s, latest: %s', name, rev, latestMod.version);
this.body = info;
};
/**
* get the special version or tag of a module
*
* GET /:name/:version
* GET /:name/:tag
*/
exports.get = function *(next) {
var name = this.params.name;
var tag = this.params.version;
exports.get = function* (next) {
var name = this.params.name || this.params[0];
var tag = this.params.version || this.params[1];
var version = semver.valid(tag);
var method = version ? 'get' : 'getByTag';
var queryLabel = version ? version : tag;
debug('%s %s with %j', method, name, this.params);
var mod = yield Module[method](name, queryLabel);
var rs = yield [
Module[method](name, queryLabel),
packageService.listMaintainers(name),
];
var mod = rs[0];
if (mod) {
common.setDownloadURL(mod.package, this);
mod.package._cnpm_publish_time = mod.publish_time;
var maintainers = rs[1];
if (maintainers.length > 0) {
mod.package.maintainers = maintainers;
}
this.body = mod.package;
return;
}
@@ -208,8 +282,6 @@ exports.get = function *(next) {
var _downloads = {};
var DOWNLOAD_TIMEOUT = ms('10m');
exports.download = function *(next) {
var name = this.params.name;
var filename = this.params.filename;
@@ -249,28 +321,13 @@ exports.download = function *(next) {
_downloads[name] = (_downloads[name] || 0) + 1;
if (typeof dist.size === 'number') {
this.set('Content-Length', dist.size);
this.length = dist.size;
}
this.set('Content-Type', mime.lookup(dist.key));
this.set('Content-Disposition', 'attachment; filename="' + filename + '"');
this.set('ETag', dist.shasum);
this.type = mime.lookup(dist.key);
this.attachment = filename;
this.etag = dist.shasum;
// use download file api
var tmpPath = path.join(config.uploadDir,
utility.randomString() + dist.key.replace(/\//g, '-'));
function cleanup() {
fs.unlink(tmpPath, utility.noop);
}
try {
yield nfs.download(dist.key, tmpPath, {timeout: DOWNLOAD_TIMEOUT});
} catch (err) {
cleanup();
this.throw(err);
}
var tarball = fs.createReadStream(tmpPath);
tarball.once('error', cleanup);
tarball.once('end', cleanup);
this.body = tarball;
this.body = yield* downloadAsReadStream(dist.key);
};
setInterval(function () {
@@ -317,104 +374,6 @@ setInterval(function () {
next();
}, 5000);
exports.upload = function *(next) {
var length = Number(this.get('content-length')) || 0;
if (!length || !this.is('application/octet-stream')) {
debug('request length or type error');
return yield *next;
}
var username = this.user.name;
var name = this.params.name;
var id = Number(this.params.rev);
var filename = this.params.filename;
var version = filename.substring(name.length + 1);
version = version.replace(/\.tgz$/, '');
// save version on pkg upload
debug('%s: upload %s, file size: %d', username, this.url, length);
var mod = yield Module.getById(id);
if (!mod) {
debug('can not get this module');
return yield* next;
}
if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not publish this module'
};
return;
}
if (mod.version !== 'next') {
// rev wrong
this.status = 403;
this.body = {
error: 'rev_wrong',
reason: 'rev not match next module'
};
return;
}
var filepath = common.getTarballFilepath(filename);
var ws = fs.createWriteStream(filepath);
var shasum = crypto.createHash('sha1');
var dataSize = 0;
var buf;
while (buf = yield coRead(this.req)) {
shasum.update(buf);
dataSize += buf.length;
yield coWrite(ws, buf);
}
ws.end();
if (dataSize !== length) {
this.status = 403;
this.body = {
error: 'size_wrong',
reason: 'Header size ' + length + ' not match download size ' + dataSize,
};
return;
}
shasum = shasum.digest('hex');
var options = {
key: common.getCDNKey(name, filename),
size: length,
shasum: shasum
};
var result;
try {
result = yield nfs.upload(filepath, options);
} catch (err) {
fs.unlink(filepath, utility.noop);
this.throw(err);
}
fs.unlink(filepath, utility.noop);
var dist = {
shasum: shasum,
size: length
};
// if nfs upload return a key, record it
if (result.url) {
dist.tarball = result.url;
} else if (result.key) {
dist.key = result.key;
dist.tarball = result.key;
}
mod.package.dist = dist;
mod.package.version = version;
debug('%s module: save file to %s, size: %d, sha1: %s, dist: %j, version: %s',
id, filepath, length, shasum, dist, version);
var updateResult = yield Module.update(mod);
this.status = 201;
this.body = {
ok: true,
rev: String(updateResult.id)
};
};
function _addDepsRelations(pkg) {
var dependencies = Object.keys(pkg.dependencies || {});
if (dependencies.length > config.maxDependencies) {
@@ -427,77 +386,13 @@ function _addDepsRelations(pkg) {
});
}
exports.updateLatest = function *(next) {
var username = this.user.name;
var name = this.params.name;
var version = semver.valid(this.params.version);
if (!version) {
this.status = 400;
this.body = {
error: 'Params Invalid',
reason: 'Invalid version: ' + this.params.version,
};
return;
}
var nextMod = yield Module.get(name, 'next');
if (!nextMod) {
debug('can not get nextMod');
return yield* next;
}
if (!common.isMaintainer(this.user, nextMod.package.maintainers)) {
this.status = 401;
this.body = {
error: 'noperms',
reason: 'Current user can not publish this module'
};
return;
}
// check version if not match pkg upload
if (nextMod.package.version !== version) {
this.status = 403;
this.body = {
error: 'version_wrong',
reason: 'version not match'
};
return;
}
var body = this.request.body;
nextMod.version = version;
nextMod.author = username;
body.dist = nextMod.package.dist;
body.maintainers = nextMod.package.maintainers;
if (!body.author) {
body.author = {
name: username,
};
}
body._publish_on_cnpm = true;
nextMod.package = body;
_addDepsRelations(body);
// reset publish time
nextMod.publish_time = Date.now();
debug('update %s:%s %j', nextMod.package.name, nextMod.package.version, nextMod.package.dist);
// change latest to version
try {
yield Module.update(nextMod);
} catch (err) {
debug('update nextMod %s error: %s', name, err);
return this.throw(err);
}
yield Module.addTag(name, 'latest', version);
nextMod.version = 'next';
var addResult = yield Module.add(nextMod);
this.status = 201;
this.body = {
ok: true,
rev: String(addResult.id)
};
};
// old flows:
// 1. add()
// 2. upload()
// 3. updateLatest()
//
// new flows: only one request
// PUT /:name
exports.addPackageAndDist = function *(next) {
// 'dist-tags': { latest: '0.0.2' },
// _attachments:
@@ -529,16 +424,36 @@ exports.addPackageAndDist = function *(next) {
tags.push([t, distTags[t]]);
}
debug('addPackageAndDist %s:%s, attachment size: %s', name, version, attachment.length);
if (tags.length === 0) {
this.status = 400;
this.body = {
error: 'invalid',
reason: 'dist-tags should not be empty'
};
return;
}
debug('%s addPackageAndDist %s:%s, attachment size: %s, maintainers: %j, distTags: %j',
username, name, version, attachment.length, versionPackage.maintainers, distTags);
var exists = yield Module.get(name, version);
var shasum;
if (exists) {
this.status = 409;
this.status = 403;
this.body = {
error: 'conflict',
reason: 'Document update conflict.'
error: 'forbidden',
reason: 'cannot modify pre-existing version: ' + version
};
return;
}
// check maintainers
var isMaintainer = yield* packageService.isMaintainer(name, username);
if (!isMaintainer) {
this.status = 403;
this.body = {
error: 'forbidden user',
reason: username + ' not authorized to modify ' + name
};
return;
}
@@ -551,11 +466,22 @@ exports.addPackageAndDist = function *(next) {
this.status = 403;
this.body = {
error: 'size_wrong',
reason: 'Attachment size ' + attachment.length + ' not match download size ' + tarballBuffer.length,
reason: 'Attachment size ' + attachment.length
+ ' not match download size ' + tarballBuffer.length,
};
return;
}
if (!distTags.latest) {
// need to check if latest tag exists or not
var latest = yield Module.getByTag(name, 'latest');
if (!latest) {
// auto add latest
tags.push(['latest', tags[0][1]]);
debug('auto add latest tag: %j', tags);
}
}
shasum = crypto.createHash('sha1');
shasum.update(tarballBuffer);
shasum = shasum.digest('hex');
@@ -607,141 +533,81 @@ exports.addPackageAndDist = function *(next) {
};
};
exports.add = function *(next) {
var username = this.user.name;
var name = this.params.name;
var pkg = this.request.body || {};
if (!common.isMaintainer(this.user, pkg.maintainers)) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not publish this module'
};
return;
}
if (pkg._attachments && Object.keys(pkg._attachments).length > 0) {
return yield exports.addPackageAndDist.call(this, next);
}
var r = yield [Module.getLatest(name), Module.get(name, 'next')];
var latestMod = r[0];
var nextMod = r[1];
if (nextMod) {
nextMod.exists = true;
} else {
nextMod = {
name: name,
version: 'next',
author: username,
package: {
name: name,
version: 'next',
description: pkg.description,
readme: pkg.readme,
maintainers: pkg.maintainers,
}
};
debug('add next module: %s', name);
var result = yield Module.add(nextMod);
nextMod.id = result.id;
}
var maintainers = latestMod && latestMod.package.maintainers.length > 0 ?
latestMod.package.maintainers : nextMod.package.maintainers;
if (!common.isMaintainer(this.user, maintainers)) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not publish this module'
};
return;
}
debug('add %s rev: %s, version: %s', name, nextMod.id, nextMod.version);
if (latestMod || nextMod.version !== 'next') {
this.status = 409;
this.body = {
error: 'conflict',
reason: 'Document update conflict.'
};
return;
}
this.status = 201;
this.body = {
ok: true,
id: name,
rev: String(nextMod.id),
};
};
exports.updateOrRemove = function *(next) {
debug('updateOrRemove module %s, %j', this.params.name, this.request.body);
// PUT /:name/-rev/:rev
exports.updateOrRemove = function* (next) {
debug('updateOrRemove module %s, %s, %j', this.url, this.params.name, this.request.body);
var body = this.request.body;
if (body.versions) {
yield *exports.removeWithVersions.call(this, next);
} else if (body.maintainers && body.maintainers.length > 0) {
yield *exports.updateMaintainers.call(this, next);
yield* exports.removeWithVersions.call(this, next);
} else if (body.maintainers) {
yield* exports.updateMaintainers.call(this, next);
} else {
yield *next;
yield* next;
}
};
exports.updateMaintainers = function *(next) {
exports.updateMaintainers = function* (next) {
var name = this.params.name;
var body = this.request.body;
debug('updateMaintainers module %s, %j', name, body);
var latestMod = yield Module.getLatest(name);
var isMaintainer = yield* packageService.isMaintainer(name, this.user.name);
if (!latestMod || !latestMod.package) {
return yield *next;
}
if (!common.isMaintainer(this.user, latestMod.package.maintainers)) {
if (!isMaintainer && !this.user.isAdmin) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not publish this module'
error: 'forbidden user',
reason: this.user.name + ' not authorized to modify ' + name
};
return;
}
var r = yield *Module.updateMaintainers(latestMod.id, body.maintainers);
var usernames = body.maintainers.map(function (user) {
return user.name;
});
if (usernames.length === 0) {
this.status = 403;
this.body = {
error: 'invalid operation',
reason: 'Can not remove all maintainers'
};
return;
}
var r = yield *packageService.updateMaintainers(name, usernames);
debug('result: %j', r);
this.status = 201;
this.body = {
ok: true,
id: name,
rev: String(latestMod.id),
rev: this.params.rev,
};
};
exports.removeWithVersions = function *(next) {
debug('removeWithVersions module %s, with info %j', this.params.name, this.request.body);
exports.removeWithVersions = function* (next) {
var username = this.user.name;
var name = this.params.name;
// left versions
var versions = this.request.body.versions || {};
debug('removeWithVersions module %s, with versions %j', name, Object.keys(versions));
// step1: list all the versions
var mods = yield Module.listByName(name);
debug('removeWithVersions module %s, left versions %j, %s mods',
name, Object.keys(versions), mods && mods.length);
if (!mods || !mods.length) {
return yield *next;
return yield* next;
}
// step2: check permission
var firstMod = mods[0];
if (!common.isMaintainer(this.user, firstMod.package.maintainers) || firstMod.name !== name) {
var isMaintainer = yield* packageService.isMaintainer(name, username);
// admin can delete the module
if (!isMaintainer && !this.user.isAdmin) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not update this module'
error: 'forbidden user',
reason: username + ' not authorized to modify ' + name
};
return;
}
@@ -801,27 +667,36 @@ exports.removeWithVersions = function *(next) {
} else {
debug('no tag need to be remove');
}
// step 7: update last modified, make sure etag change
yield* Module.updateLastModified(name);
this.status = 201;
this.body = { ok: true };
};
exports.removeTar = function *(next) {
exports.removeTar = function* (next) {
debug('remove tarball with filename: %s, id: %s', this.params.filename, this.params.rev);
var id = Number(this.params.rev);
var filename = this.params.filename;
var name = this.params.name;
var username = this.user.name;
var mod = yield Module.getById(id);
if (!mod) {
if (isNaN(id)) {
return yield* next;
}
if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
var mod = yield Module.getById(id);
if (!mod || mod.name !== name) {
return yield* next;
}
var isMaintainer = yield* packageService.isMaintainer(name, username);
if (!isMaintainer && !this.user.isAdmin) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not delete this tarball'
error: 'forbidden user',
reason: username + ' not authorized to modify ' + name
};
return;
}
@@ -831,10 +706,10 @@ exports.removeTar = function *(next) {
this.body = { ok: true };
};
exports.removeAll = function *(next) {
exports.removeAll = function* (next) {
debug('remove all the module with name: %s, id: %s', this.params.name, this.params.rev);
// var id = Number(this.params.rev);
var name = this.params.name;
var username = this.user.name;
var mods = yield Module.listByName(name);
debug('removeAll module %s: %d', name, mods.length);
@@ -843,11 +718,13 @@ exports.removeAll = function *(next) {
return yield* next;
}
if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
var isMaintainer = yield* packageService.isMaintainer(name, username);
// admin can delete the module
if (!isMaintainer && !this.user.isAdmin) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not delete this tarball'
error: 'forbidden user',
reason: username + ' not authorized to modify ' + name
};
return;
}
@@ -855,16 +732,24 @@ exports.removeAll = function *(next) {
yield [Module.removeByName(name), Module.removeTags(name)];
var keys = [];
for (var i = 0; i < mods.length; i++) {
var key = urlparse(mods[i].dist_tarball).path;
var row = mods[i];
var dist = row.package.dist;
var key = dist.key;
if (!key) {
key = urlparse(dist.tarball).pathname;
}
key && keys.push(key);
}
try {
yield keys.map(function (key) {
return nfs.remove(key);
});
} catch (err) {
// ignore error here
if (keys.length > 0) {
try {
yield keys.map(function (key) {
return nfs.remove(key);
});
} catch (err) {
// ignore error here
}
}
this.body = { ok: true };
};
@@ -893,8 +778,12 @@ function parseModsForList(updated, mods, ctx) {
exports.listAllModules = function *() {
var updated = Date.now();
var mods = yield Module.listSince(0);
this.body = parseModsForList(updated, mods, this);
var mods = yield Module.listAllNames();
var result = { _updated: updated };
mods.forEach(function (mod) {
result[mod.name] = true;
});
this.body = result;
};
exports.listAllModulesSince = function *() {
@@ -912,7 +801,12 @@ exports.listAllModulesSince = function *() {
var startkey = Number(query.startkey) || 0;
var updated = Date.now();
var mods = yield Module.listSince(startkey);
this.body = parseModsForList(updated, mods, this);
var result = { _updated: updated };
mods.forEach(function (mod) {
result[mod.name] = true;
});
this.body = result;
};
exports.listAllModuleNames = function *() {
@@ -920,3 +814,60 @@ exports.listAllModuleNames = function *() {
return m.name;
});
};
// PUT /:name/:tag
exports.updateTag = function* () {
var version = this.request.body;
var tag = this.params.tag;
var name = this.params.name;
debug('updateTag: %s %s to %s', name, version, tag);
if (!version) {
this.status = 400;
this.body = {
error: 'version_missed',
reason: 'version not found'
};
return;
}
if (!semver.valid(version)) {
this.status = 403;
var reason = util.format('setting tag %s to invalid version: %s: %s/%s',
tag, version, name, tag);
this.body = {
error: 'forbidden',
reason: reason
};
return;
}
var mod = yield Module.get(name, version);
if (!mod) {
this.status = 403;
var reason = util.format('setting tag %s to unknown version: %s: %s/%s',
tag, version, name, tag);
this.body = {
error: 'forbidden',
reason: reason
};
return;
}
// check permission
var isMaintainer = yield* packageService.isMaintainer(name, this.user.name);
if (!isMaintainer) {
this.status = 403;
this.body = {
error: 'forbidden user',
reason: this.user.name + ' not authorized to modify ' + name
};
return;
}
yield Module.addTag(name, tag, version);
this.status = 201;
this.body = {
ok: true
};
};

View File

@@ -15,7 +15,7 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:controllers:registry');
var debug = require('debug')('cnpmjs.org:controllers:registry:user');
var utility = require('utility');
var crypto = require('crypto');
var User = require('../../proxy/user');

View File

@@ -1,5 +1,5 @@
/**!
* cnpmjs.org - controllers/download.js
* cnpmjs.org - controllers/sync.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
@@ -14,14 +14,16 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:controllers:sync');
var Log = require('../proxy/module_log');
var SyncModuleWorker = require('../proxy/sync_module_worker');
exports.sync = function *() {
exports.sync = function* () {
var username = this.user.name || 'anonymous';
var name = this.params.name;
var name = this.params.name || this.params[0];
var publish = this.query.publish === 'true';
var noDep = this.query.nodeps === 'true';
debug('sync %s with query: %j', name, this.query);
if (publish && !this.user.isAdmin) {
this.status = 403;
this.body = {
@@ -37,9 +39,10 @@ exports.sync = function *() {
};
var result = yield SyncModuleWorker.sync(name, username, options);
debug('sync %s got %j', name, result);
// friendly 404 reason info
if (result.staticCache === 404) {
if (result.statusCode === 404) {
this.status = 404;
this.body = {
ok: false,
@@ -48,7 +51,7 @@ exports.sync = function *() {
return;
}
if (!result.ok) {
this.status = result.statusCode;
this.status = result.statusCode || 500;
this.body = result.pkg;
return;
}
@@ -59,8 +62,9 @@ exports.sync = function *() {
};
};
exports.getSyncLog = function *(next) {
var logId = this.params.id;
exports.getSyncLog = function* (next) {
// params: [$name, $id] on scope package
var logId = this.params.id || this.params[1];
var offset = Number(this.query.offset) || 0;
var row = yield Log.get(logId);
if (!row) {

View File

@@ -15,13 +15,12 @@
* Module dependencies.
*/
var microtime = require('microtime');
var Total = require('../proxy/total');
var Download = require('./download');
var version = require('../package.json').version;
var config = require('../config');
var startTime = '' + microtime.now();
var startTime = '' + Date.now();
exports.show = function *() {
var r = yield [Total.get(), Download.total()];

46
controllers/utils.js Normal file
View File

@@ -0,0 +1,46 @@
/**!
* cnpmjs.org - controllers/utils.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:controllers:utils');
var path = require('path');
var fs = require('fs');
var utility = require('utility');
var ms = require('ms');
var nfs = require('../common/nfs');
var config = require('../config');
var DOWNLOAD_TIMEOUT = ms('10m');
exports.downloadAsReadStream = function* (key) {
var tmpPath = path.join(config.uploadDir,
utility.randomString() + key.replace(/\//g, '-'));
function cleanup() {
debug('cleanup %s', tmpPath);
fs.unlink(tmpPath, utility.noop);
}
debug('downloadAsReadStream() %s to %s', key, tmpPath);
try {
yield nfs.download(key, tmpPath, {timeout: DOWNLOAD_TIMEOUT});
} catch (err) {
debug('downloadAsReadStream() %s to %s error: %s', key, tmpPath, err.stack);
cleanup();
throw err;
}
var tarball = fs.createReadStream(tmpPath);
tarball.once('error', cleanup);
tarball.once('end', cleanup);
return tarball;
};

View File

@@ -14,10 +14,72 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:controllers:web:dist');
var mime = require('mime');
var Dist = require('../../proxy/dist');
var config = require('../../config');
var downloadAsReadStream = require('../utils').downloadAsReadStream;
exports.redirect = function *(next) {
function padding(max, current, pad) {
pad = pad || ' ';
var left = max - current;
var str = '';
for (var i = 0; i < left; i++) {
str += pad;
}
return str;
}
exports.list = function* (next) {
var params = this.params;
var url = config.disturl + (params[0] || '/');
this.redirect(url);
var url = params[0];
if (!url) {
// GET /dist => /dist/
return this.redirect('/dist/');
}
var isDir = url[url.length - 1] === '/';
if (!isDir) {
return yield* download.call(this, next);
}
var items = yield* Dist.listdir(url);
if (url === '/') {
// phantomjs/
items.push({
name: 'phantomjs/',
date: '',
});
}
yield this.render('dist', {
title: 'Mirror index of ' + config.disturl + url,
disturl: config.disturl,
dirname: url,
items: items,
padding: padding
});
};
function* download(next) {
var fullname = this.params[0];
var info = yield* Dist.getfile(fullname);
debug('download %s got %j', fullname, info);
if (!info || !info.url) {
return yield* next;
}
if (info.url.indexOf('http') === 0) {
return this.redirect(info.url);
}
// download it from nfs
if (typeof info.size === 'number') {
this.length = info.size;
}
this.type = mime.lookup(info.url);
this.attachment = info.name;
this.etag = info.sha1;
this.body = yield* downloadAsReadStream(info.url);
}

View File

@@ -14,6 +14,8 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:controllers:web:package');
var bytes = require('bytes');
var giturl = require('giturl');
var moment = require('moment');
var eventproxy = require('eventproxy');
@@ -29,11 +31,15 @@ var Log = require('../../proxy/module_log');
var ModuleDeps = require('../../proxy/module_deps');
var setDownloadURL = require('../../lib/common').setDownloadURL;
var ModuleStar = require('../../proxy/module_star');
var packageService = require('../../services/package');
exports.display = function *(next) {
exports.display = function* (next) {
var params = this.params;
var name = params.name;
var tag = params.version;
// normal: {name: $name, version: $version}
// scope: [$name, $version]
var name = params.name || params[0];
var tag = params.version || params[1];
debug('display %s with %j', name, params);
var getPackageMethod;
var getPackageArgs;
@@ -50,6 +56,7 @@ exports.display = function *(next) {
down.total(name),
ModuleDeps.list(name),
ModuleStar.listUsers(name),
packageService.listMaintainers(name)
];
var pkg = r[0];
var download = r[1];
@@ -57,6 +64,7 @@ exports.display = function *(next) {
return item.deps;
});
var users = r[3];
var maintainers = r[4];
if (!pkg || !pkg.package) {
return yield* next;
@@ -70,11 +78,15 @@ exports.display = function *(next) {
pkg.readme = pkg.description || '';
}
if (maintainers.length > 0) {
pkg.maintainers = maintainers;
}
if (pkg.maintainers) {
for (var i = 0; i < pkg.maintainers.length; i++) {
var maintainer = pkg.maintainers[i];
if (maintainer.email) {
maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, true);
}
}
}
@@ -87,7 +99,7 @@ exports.display = function *(next) {
for (var i = 0; i < pkg.contributors.length; i++) {
var contributor = pkg.contributors[i];
if (contributor.email) {
contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, true);
}
if (config.packagePageContributorSearch || !contributor.url) {
contributor.url = '/~' + encodeURIComponent(contributor.name);
@@ -108,6 +120,10 @@ exports.display = function *(next) {
pkg.dependents = dependents;
if (pkg.dist) {
pkg.dist.size = bytes(pkg.dist.size || 0);
}
yield this.render('package', {
title: 'Package - ' + pkg.name,
package: pkg,
@@ -117,22 +133,34 @@ exports.display = function *(next) {
exports.search = function *(next) {
var params = this.params;
var word = params.word;
var word = params.word || params[0];
debug('search %j', word);
var result = yield Module.search(word);
var match = null;
for (var i = 0; i < result.searchMatchs.length; i++) {
var p = result.searchMatchs[i];
if (p.name === word) {
match = p;
break;
}
}
// return a json result
if (this.query && this.query.type === 'json') {
this.body = {
keyword: word,
match: match,
packages: result.searchMatchs,
keywords: result.keywordMatchs
keywords: result.keywordMatchs,
};
this.type = 'application/json; charset=utf-8';
return;
}
yield this.render('search', {
title: 'Keyword - ' + word,
keyword: word,
match: match,
packages: result.searchMatchs,
keywords: result.keywordMatchs,
});
@@ -169,8 +197,8 @@ exports.rangeSearch = function *(next) {
};
};
exports.displaySync = function *(next) {
var name = this.params.name || this.query.name;
exports.displaySync = function* (next) {
var name = this.params.name || this.params[0] || this.query.name;
yield this.render('sync', {
name: name,
title: 'Sync - ' + name

View File

@@ -24,6 +24,18 @@ var childProcess = require('child_process');
var syncPath = path.join(__dirname, 'sync');
if (config.enableCluster) {
forkWorker();
if (config.syncModel !== 'none') {
forkSyncer();
}
} else {
require(workerPath);
if (config.syncModel !== 'none') {
require(syncPath);
}
}
function forkWorker() {
cluster.setupMaster({
exec: workerPath
});
@@ -50,9 +62,16 @@ if (config.enableCluster) {
for (var i = 0; i < config.numCPUs; i++) {
cluster.fork();
}
childProcess.fork(syncPath);
} else {
require(workerPath);
require(syncPath);
}
function forkSyncer() {
var syncer = childProcess.fork(syncPath);
syncer.on('exit', function (code, signal) {
var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)',
syncer.pid, code, signal, syncer.stdout, syncer.stderr));
err.name = 'SyncerWorkerDiedError';
console.error('[%s] [master:%s] syncer exit: %s: %s',
Date(), process.pid, err.name, err.message);
setTimeout(forkSyncer, 1000);
});
}

View File

@@ -40,6 +40,16 @@ CREATE TABLE `module_star` (
KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
CREATE TABLE `module_maintainer` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
PRIMARY KEY (`id`),
UNIQUE KEY `user_module_name` (`user`,`name`),
KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module maintainers';
CREATE TABLE `module` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
@@ -87,10 +97,23 @@ CREATE TABLE `tag` (
`version` varchar(30) NOT NULL COMMENT 'module version',
`module_id` bigint(20) unsigned NOT NULL COMMENT 'module id',
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`, `tag`)
UNIQUE KEY `name` (`name`, `tag`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module tag';
-- ALTER TABLE `tag` ADD `module_id` BIGINT( 20 ) UNSIGNED NOT NULL;
-- ALTER TABLE `tag` CHANGE `name` `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
-- ALTER TABLE `tag` ADD KEY `gmt_modified` (`gmt_modified`);
CREATE TABLE `module_unpublished` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
`package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'base info: tags, time, maintainers, description, versions',
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module unpublished info';
CREATE TABLE `total` (
`name` varchar(100) NOT NULL COMMENT 'total name',
@@ -137,3 +160,30 @@ CREATE TABLE `module_deps` (
UNIQUE KEY `name_deps` (`name`,`deps`),
KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';
CREATE TABLE `dist_dir` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`name` varchar(200) NOT NULL COMMENT 'user name',
`parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
`date` varchar(20) COMMENT '02-May-2014 01:06',
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`parent`, `name`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
CREATE TABLE `dist_file` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`name` varchar(100) NOT NULL COMMENT 'user name',
`parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
`date` varchar(20) COMMENT '02-May-2014 01:06',
`size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',
`sha1` varchar(40) COMMENT 'sha1 hex value',
`url` varchar(2048),
PRIMARY KEY (`id`),
UNIQUE KEY `fullname` (`parent`, `name`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist file info';

9
docs/new.sql Normal file
View File

@@ -0,0 +1,9 @@
CREATE TABLE `module_maintainer` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
PRIMARY KEY (`id`),
UNIQUE KEY `user_module_name` (`user`,`name`),
KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module maintainers';

View File

@@ -1,24 +1,13 @@
# cnpmjs.org: Private npm registry and web for Enterprise
# cnpmjs.org: Private npm registry and web for Company
[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
## What is this?
> Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
* @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
## Install your private npm registry
@see [Install and Get Started](/install).
So `cnpm` is meaning: **Company npm**.
## Registry
* Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
* Current [cnpmjs.org](/) version: <span id="app-version"></span>
* Mirror of [nodejs.org/dist](http://nodejs.org/dist): [/dist mirror](/dist)
* Mirror of [phantomjs downloads](https://bitbucket.org/ariya/phantomjs/downloads): [phantomjs mirror](/dist/phantomjs/)
<table class="downloads">
<tbody>
@@ -115,9 +104,15 @@ $(function () {
});
</script>
## cnpm cli
## Usage
alias it:
use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
```
npm install -g cnpm --registry=http://r.cnpmjs.org
```
Or you can alias NPM to use it:
```bash
alias cnpm="npm --registry=http://r.cnpmjs.org \
@@ -132,12 +127,6 @@ $ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
--userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
```
Or you can just use our `cnpm` cli:
```bash
$ npm install cnpm -g
```
### install
Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
@@ -181,33 +170,15 @@ $ cnpm info cnpm
@see Github [Issues](https://github.com/cnpm/cnpmjs.org/issues)
## Authors
## Histories
Release [History](/history).
```bash
$ git summary
project : cnpmjs.org
repo age : 7 weeks
active : 132 days
commits : 315
files : 88
authors :
190 fengmk2 60.3%
122 dead_horse 38.7%
2 4simple 0.6%
1 Alsotang 0.3%
```
## npm and cnpm relation
![npm&cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=383&h=284)
## 捐赠 Donate
如果您觉得 [cnpmjs.org] 对您有帮助,欢迎请作者一杯咖啡.
如果您觉得 [cnpmjs.org](/) 对您有帮助,欢迎请作者一杯咖啡.
[![Donate](https://img.alipay.com/sys/personalprod/style/mc/btn-index.png)](https://me.alipay.com/imk2)
[cnpmjs.org]: http://cnpmjs.org/
[registry.cnpmjs.org]: http://registry.cnpmjs.org/

View File

@@ -21,6 +21,7 @@ var util = require('util');
exports.getTarballFilepath = function (filename) {
// ensure download file path unique
// TODO: not only .tgz, and also other extname
var name = filename.replace(/\.tgz$/, '.' + crypto.randomBytes(16).toString('hex') + '.tgz');
return path.join(config.uploadDir, name);
};

View File

@@ -17,9 +17,13 @@
module.exports = function *notFound(next) {
yield *next;
if (this.status) {
if (this.status && this.status !== 404) {
return;
}
if (this.body && this.body.name) {
return;
}
this.status = 404;
this.body = {
error: 'not_found',

29
middleware/static.js Normal file
View File

@@ -0,0 +1,29 @@
/**!
* cnpmjs.org - middleware/static.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var path = require('path');
var middlewares = require('koa-middlewares');
var config = require('../config');
var staticDir = path.join(path.dirname(__dirname), 'public');
module.exports = middlewares.staticCache(staticDir, {
buffer: config.debug ? false : true,
maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
alas: {
'/favicon.ico': '/favicon.png'
}
});

View File

@@ -20,21 +20,22 @@ var config = require('../config');
* this.allowSync - allow sync triggle by cnpm install
*/
module.exports = function *syncByInstall(next) {
module.exports = function* syncByInstall(next) {
if (!config.syncByInstall || !config.enablePrivate) {
// only config.enablePrivate should enable sync on install
return yield *next;
return yield* next;
}
// request not by node, consider it request from web
var ua = this.get('user-agent');
if (!ua || ua.indexOf('node') < 0) {
return yield *next;
return yield* next;
}
// if request with `/xxx?write=true`, meaning the read request using for write
if (this.query.write) {
return yield *next;
return yield* next;
}
this.allowSync = true;
yield *next;
yield* next;
};

View File

@@ -14,20 +14,36 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:middleware:web_not_found');
module.exports = function *notFound(next) {
yield *next;
if (this.status) {
if (this.status && this.status !== 404) {
return;
}
if (this.body) {
return;
}
var m = /^\/([\w\-\_\.]+)\/?$/.exec(this.url);
debug('%s match %j', this.url, m);
if (m) {
return this.redirect('/package/' + m[1]);
}
// package not found
m = /\/package\/([\w\-\_\.]+)\/?$/.exec(this.url);
if (m) {
var name = m[1];
this.status = 404;
yield* this.render('404', {
title: 'Package - ' + name,
name: name
});
return;
}
this.status = 404;
this.type = 'text/html';
this.charset = 'utf-8';
this.body = 'Cannot GET ' + this.path;
};

View File

@@ -1,6 +1,6 @@
{
"name": "cnpmjs.org",
"version": "0.3.11",
"version": "0.8.0",
"description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
"main": "index.js",
"scripts": {
@@ -9,58 +9,58 @@
"status": "./bin/nodejsctl status",
"stop": "./bin/nodejsctl stop"
},
"config": {
"cov": {
"threshold": 83
}
},
"dependencies": {
"co": "3.0.5",
"bytes": "1.0.0",
"cheerio": "0.17.0",
"co": "3.0.6",
"co-defer": "0.1.0",
"co-gather": "0.0.1",
"co-read": "0.0.1",
"co-read": "0.1.0",
"co-redis": "1.1.0",
"co-urllib": "0.1.3",
"co-urllib": "0.2.3",
"co-write": "0.3.0",
"debug": "0.7.4",
"eventproxy": "0.3.0",
"giturl": "0.0.2",
"graceful": "0.0.6",
"copy-to": "1.0.1",
"debug": "1.0.4",
"error-formater": "1.0.3",
"eventproxy": "0.3.1",
"giturl": "0.0.3",
"graceful": "0.1.0",
"gravatar": "1.0.6",
"humanize-number": "0.0.2",
"koa": "0.5.1",
"koa-limit": "1.0.0",
"koa": "0.8.1",
"koa-limit": "1.0.2",
"koa-markdown": "0.0.3",
"koa-middlewares": "0.0.12",
"logfilestream": "0.1.0",
"koa-middlewares": "1.2.0",
"marked": "0.3.2",
"microtime": "0.5.1",
"mime": "1.2.11",
"mkdirp": "0.3.5",
"moment": "2.5.1",
"mini-logger": "0.3.0",
"mkdirp": "0.5.0",
"moment": "2.7.0",
"ms": "0.6.2",
"multiline": "0.2.0",
"mysql": "2.1.1",
"nodemailer": "0.6.1",
"qn": "0.2.1",
"multiline": "0.3.4",
"mysql": "2.4.1",
"nodemailer": "0.7.1",
"qn": "0.2.2",
"ready": "0.1.1",
"redis": "0.10.1",
"semver": "2.2.1",
"thunkify-wrap": "0.1.1",
"utility": "0.1.11"
"redis": "0.11.0",
"semver": "2.3.1",
"thunkify-wrap": "1.0.1",
"utility": "0.1.16"
},
"devDependencies": {
"autod": ">=0.0.13",
"autod": "~0.2.0",
"chunkstream": "0.0.1",
"co-mocha": "0.0.2",
"contributors": "*",
"cov": "*",
"istanbul": "git://github.com/gotwarlost/istanbul.git#harmony",
"istanbul-harmony": "*",
"jshint": "*",
"mm": "0.2.1",
"mocha": "*",
"pedding": "0.0.3",
"should": "3.1.3",
"supertest": "0.9.2"
"pedding": "1.0.0",
"should": "4.0.4",
"should-http": "0.0.1",
"supertest": "0.13.0"
},
"homepage": "https://github.com/cnpm/cnpmjs.org",
"repository": {

85
proxy/dist.js Normal file
View File

@@ -0,0 +1,85 @@
/**!
* cnpmjs.org - proxy/dist.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var path = require('path');
var multiline = require('multiline');
var mysql = require('../common/mysql');
var SAVE_FILE_SQL = multiline(function () {;/*
INSERT INTO
dist_file(gmt_create, gmt_modified, name, parent, date, size, url, sha1)
VALUES
(now(), now(), ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
name=VALUES(name),
parent=VALUES(parent),
date=VALUES(date),
size=VALUES(size),
url=VALUES(url),
sha1=VALUES(sha1);
*/});
exports.savefile = function* (info) {
return yield mysql.query(SAVE_FILE_SQL, [
info.name, info.parent, info.date, info.size, info.url, info.sha1
]);
};
var SAVE_DIR_SQL = multiline(function () {;/*
INSERT INTO
dist_dir(gmt_create, gmt_modified, name, parent, date)
VALUES
(now(), now(), ?, ?, ?)
ON DUPLICATE KEY UPDATE
name=VALUES(name),
parent=VALUES(parent),
date=VALUES(date);
*/});
exports.savedir = function* (info) {
return yield mysql.query(SAVE_DIR_SQL, [
info.name, info.parent, info.date
]);
};
var LIST_DIRS_SQL = multiline(function () {;/*
SELECT name, parent, date FROM dist_dir WHERE parent = ?;
*/});
var LIST_FILES_SQL = multiline(function () {;/*
SELECT name, parent, date, size, url, sha1
FROM dist_file WHERE parent = ?;
*/});
exports.listdir = function* (name) {
var rs = yield [
mysql.query(LIST_DIRS_SQL, [name]),
mysql.query(LIST_FILES_SQL, [name]),
];
return rs[0].concat(rs[1]);
};
var GET_FILE_SQL = multiline(function () {;/*
SELECT name, parent, date, url, size, sha1 FROM dist_file WHERE parent = ? AND name = ?;
*/});
exports.getfile = function* (fullname) {
var name = path.basename(fullname);
var parent = path.dirname(fullname);
if (parent !== '/') {
parent += '/';
}
return yield mysql.queryOne(GET_FILE_SQL, [parent, name]);
};

View File

@@ -19,7 +19,7 @@ var config = require('../config');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var PLUS_SQL = multiline(function () {/*
var PLUS_SQL = multiline(function () {;/*
INSERT INTO
download_total(gmt_create, gmt_modified, date, name, count)
VALUES
@@ -33,7 +33,7 @@ exports.plusTotal = function (data, callback) {
mysql.query(PLUS_SQL, [data.date, data.name, data.count], callback);
};
var SELECT_ONE_TOTAL_SQL = multiline(function () {/*
var SELECT_ONE_TOTAL_SQL = multiline(function () {;/*
SELECT
date, count
FROM
@@ -45,7 +45,7 @@ exports.getModuleTotal = function (name, start, end, callback) {
mysql.query(SELECT_ONE_TOTAL_SQL, [start, end, name], callback);
};
var SELECT_ALL_TOTAL_SQL = multiline(function () {/*
var SELECT_ALL_TOTAL_SQL = multiline(function () {;/*
SELECT
date, sum(count) AS count
FROM

View File

@@ -21,10 +21,7 @@ var config = require('../config');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, \
version, description, package, dist_tarball, dist_shasum, dist_size';
var INSERT_MODULE_SQL = multiline(function () {/*
var INSERT_MODULE_SQL = multiline(function () {;/*
INSERT INTO
module(gmt_create, gmt_modified, publish_time, author, name, version,
package, dist_tarball, dist_shasum, dist_size, description)
@@ -96,7 +93,7 @@ exports.add = function (mod, callback) {
});
};
var GET_KEYWORD_SQL = multiline(function () {/*
var GET_KEYWORD_SQL = multiline(function () {;/*
SELECT
keyword
FROM
@@ -119,7 +116,7 @@ exports.getKeywords = function (name, callback) {
});
};
var ADD_KEYWORD_SQL = multiline(function () {/*
var ADD_KEYWORD_SQL = multiline(function () {;/*
INSERT INTO
module_keyword(gmt_create, keyword, name, description)
VALUES
@@ -152,7 +149,7 @@ exports.addKeywords = function (name, description, keywords, callback) {
});
};
var UPDATE_DESC_SQL = multiline(function () {/*
var UPDATE_DESC_SQL = multiline(function () {;/*
UPDATE
module
SET
@@ -164,7 +161,7 @@ exports.updateDescription = function (id, description, callback) {
mysql.query(UPDATE_DESC_SQL, [description, id], callback);
};
var UPDATE_PACKAGE_SQL = multiline(function () {/*
var UPDATE_PACKAGE_SQL = multiline(function () {;/*
UPDATE
module
SET
@@ -184,19 +181,7 @@ exports.updateReadme = function (id, readme, callback) {
});
};
var UPDATE_DIST_SQL = multiline(function () {/*
UPDATE
module
SET
publish_time=?,
version=?,
package=?,
dist_tarball=?,
dist_shasum=?,
dist_size=?
WHERE
id=?;
*/});
var UPDATE_DIST_SQL = 'UPDATE module SET ? WHERE id=?';
exports.update = function (mod, callback) {
var pkg;
try {
@@ -205,8 +190,18 @@ exports.update = function (mod, callback) {
return callback(e);
}
var dist = mod.package.dist;
var arg = {
publish_time: mod.publish_time,
version: mod.version,
package: pkg,
dist_tarball: dist.tarball,
dist_shasum: dist.shasum,
dist_size: dist.size
};
mysql.query(UPDATE_DIST_SQL,
[mod.publish_time, mod.version, pkg, dist.tarball, dist.shasum, dist.size, mod.id],
[arg, mod.id],
function (err, result) {
if (err) {
return callback(err);
@@ -235,7 +230,7 @@ function stringifyPackage(pkg) {
}
var SELECT_MODULE_BY_ID_SQL = multiline(function () {/*
var SELECT_MODULE_BY_ID_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
@@ -260,7 +255,7 @@ exports.getById = function (id, callback) {
});
};
var SELECT_MODULE_SQL = multiline(function () {/*
var SELECT_MODULE_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
@@ -284,7 +279,7 @@ exports.get = function (name, version, callback) {
});
};
var SELECT_MODULE_ID_SQL = multiline(function () {/*
var SELECT_MODULE_ID_SQL = multiline(function () {;/*
SELECT
id
FROM
@@ -292,7 +287,7 @@ var SELECT_MODULE_ID_SQL = multiline(function () {/*
WHERE
name=? AND version=?;
*/});
var INSERT_TAG_SQL = multiline(function () {/*
var INSERT_TAG_SQL = multiline(function () {;/*
INSERT INTO
tag(gmt_create, gmt_modified, name, tag, version, module_id)
VALUES
@@ -319,7 +314,7 @@ exports.addTag = function (name, tag, version, callback) {
});
};
var SELECT_TAG_SQL = multiline(function () {/*
var SELECT_TAG_SQL = multiline(function () {;/*
SELECT
tag, version, gmt_modified, module_id
FROM
@@ -336,7 +331,7 @@ exports.getByTag = function (name, tag, callback) {
});
};
var DELETE_TAGS_SQL = multiline(function () {/*
var DELETE_TAGS_SQL = multiline(function () {;/*
DELETE FROM
tag
WHERE
@@ -346,17 +341,17 @@ exports.removeTags = function (name, callback) {
mysql.query(DELETE_TAGS_SQL, [name], callback);
};
var DELETE_TAGS_BY_IDS_SQL = multiline(function () {/*
var DELETE_TAGS_BY_IDS_SQL = multiline(function () {;/*
DELETE FROM
tag
WHERE
id in (?);
id IN (?);
*/});
exports.removeTagsByIds = function (ids, callback) {
mysql.query(DELETE_TAGS_BY_IDS_SQL, [ids], callback);
};
var SELECT_ALL_TAGS_SQL = multiline(function () {/*
var SELECT_ALL_TAGS_SQL = multiline(function () {;/*
SELECT
id, tag, version, gmt_modified, module_id
FROM
@@ -368,7 +363,7 @@ exports.listTags = function (name, callback) {
mysql.query(SELECT_ALL_TAGS_SQL, [name], callback);
};
var SELECT_LATEST_MODULE_SQL = multiline(function () {/*
var SELECT_LATEST_MODULE_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
@@ -403,7 +398,7 @@ exports.getLatest = function (name, callback) {
});
};
var LIST_MODULE_SQL = multiline(function () {/*
var LIST_MODULE_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
@@ -433,41 +428,29 @@ exports.listByName = function (name, callback) {
});
};
var LIST_SINCE_SQLS = [];
LIST_SINCE_SQLS.push(multiline(function () {/*
var LIST_SINCE_SQL = multiline(function () {;/*
SELECT
module_id
distinct(name)
FROM
tag
WHERE
tag="latest" AND gmt_modified>?;
*/}));
LIST_SINCE_SQLS.push(multiline(function () {/*
SELECT
name, package
FROM
module
WHERE
id IN (?);
*/}));
gmt_modified > ?;
*/});
exports.listSince = function (start, callback) {
var ep = eventproxy.create();
ep.fail(callback);
mysql.query(LIST_SINCE_SQLS[0], [new Date(start)], ep.done(function (rows) {
if (!rows || rows.length === 0) {
return callback(null, []);
}
ep.emit('ids', rows.map(function (r) {
return r.module_id;
}));
}));
ep.once('ids', function (ids) {
mysql.query(LIST_SINCE_SQLS[1], [ids], callback);
});
mysql.query(LIST_SINCE_SQL, [new Date(start)], callback);
};
var LIST_SHORT_SQL = multiline(function () {/*
var LIST_ALL_NAME_SQL = multiline(function () {;/*
SELECT
distinct(name)
FROM
module;
*/});
exports.listAllNames = function (callback) {
mysql.query(LIST_ALL_NAME_SQL, [], callback);
};
var LIST_SHORT_SQL = multiline(function () {;/*
SELECT
distinct(name)
FROM
@@ -479,7 +462,7 @@ exports.listShort = function (callback) {
mysql.query(LIST_SHORT_SQL, callback);
};
var LIST_ALL_MODULE_NAMES_SQL = multiline(function () {/*
var LIST_ALL_MODULE_NAMES_SQL = multiline(function () {;/*
SELECT
distinct(name)
FROM
@@ -491,7 +474,7 @@ exports.listAllModuleNames = function (callback) {
mysql.query(LIST_ALL_MODULE_NAMES_SQL, callback);
};
var DELETE_MODULE_BY_NAME_SQL = multiline(function () {/*
var DELETE_MODULE_BY_NAME_SQL = multiline(function () {;/*
DELETE FROM
module
WHERE
@@ -501,7 +484,7 @@ exports.removeByName = function (name, callback) {
mysql.query(DELETE_MODULE_BY_NAME_SQL, [name], callback);
};
var DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL = multiline(function () {/*
var DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL = multiline(function () {;/*
DELETE FROM
module
WHERE
@@ -512,7 +495,7 @@ exports.removeByNameAndVersions = function (name, versions, callback) {
};
var LIST_BY_AUTH_SQLS = [];
LIST_BY_AUTH_SQLS.push(multiline(function () {/*
LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
SELECT
distinct(name) AS name
FROM
@@ -524,7 +507,7 @@ LIST_BY_AUTH_SQLS.push(multiline(function () {/*
LIMIT
100;
*/}));
LIST_BY_AUTH_SQLS.push(multiline(function () {/*
LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
SELECT
module_id
FROM
@@ -532,7 +515,7 @@ LIST_BY_AUTH_SQLS.push(multiline(function () {/*
WHERE
tag="latest" AND name IN (?);
*/}));
LIST_BY_AUTH_SQLS.push(multiline(function () {/*
LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
SELECT
name, description
FROM
@@ -568,7 +551,7 @@ exports.listByAuthor = function (author, callback) {
});
};
var SEARCH_MODULES_SQL = multiline(function () {/*
var SEARCH_MODULES_SQL = multiline(function () {;/*
SELECT
module_id
FROM
@@ -580,7 +563,7 @@ var SEARCH_MODULES_SQL = multiline(function () {/*
LIMIT
?;
*/});
var SEARCH_MODULES_BY_KEYWORD_SQL = multiline(function () {/*
var SEARCH_MODULES_BY_KEYWORD_SQL = multiline(function () {;/*
SELECT
name, description
FROM
@@ -592,7 +575,7 @@ var SEARCH_MODULES_BY_KEYWORD_SQL = multiline(function () {/*
LIMIT
?;
*/});
var QUERY_MODULES_BY_ID_SQL = multiline(function () {/*
var QUERY_MODULES_BY_ID_SQL = multiline(function () {;/*
SELECT
name, description
FROM
@@ -665,9 +648,31 @@ exports.search = function (word, options, callback) {
thunkify(exports);
exports.updateMaintainers = function *(id, maintainers) {
var mod = yield exports.getById(id);
mod.package.maintainers = maintainers;
var pkg = stringifyPackage(mod.package);
return yield mysql.query(UPDATE_PACKAGE_SQL, [pkg, id]);
var GET_LAST_MODIFIED_MODULE_SQL = multiline(function () {;/*
SELECT
id, gmt_modified
FROM
module
WHERE
name=?
ORDER BY
gmt_modified DESC
LIMIT 1;
*/});
exports.getLastModified = function* (name) {
var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
return row && row.gmt_modified;
};
var UPDATE_LAST_MODIFIED_SQL = 'UPDATE module SET gmt_modified=now() WHERE id=?;';
exports.updateLastModified = function* (name) {
var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
if (row) {
yield mysql.query(UPDATE_LAST_MODIFIED_SQL, [row.id]);
}
};
var DELETE_TAGS_BY_NAMES_SQL = 'DELETE FROM tag WHERE name=? AND tag IN (?);';
exports.removeTagsByNames = function* (moduleName, tagNames) {
return yield mysql.query(DELETE_TAGS_BY_NAMES_SQL, [moduleName, tagNames]);
};

View File

@@ -18,7 +18,7 @@ var thunkify = require('thunkify-wrap');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var LIST_DEPS_SQL = multiline(function () {/*
var LIST_DEPS_SQL = multiline(function () {;/*
SELECT
deps
FROM
@@ -30,7 +30,7 @@ exports.list = function (name, callback) {
mysql.query(LIST_DEPS_SQL, [name], callback);
};
var INSERT_DEPS_SQL = multiline(function () {/*
var INSERT_DEPS_SQL = multiline(function () {;/*
INSERT INTO
module_deps(gmt_create, name, deps)
VALUES
@@ -45,7 +45,7 @@ exports.add = function (name, deps, callback) {
});
};
var DELETE_DEPS_SQL = multiline(function () {/*
var DELETE_DEPS_SQL = multiline(function () {;/*
DELETE FROM
module_deps
WHERE

View File

@@ -16,16 +16,18 @@
var thunkify = require('thunkify-wrap');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var INSERT_LOG_SQL = multiline(function () {/*
INSERT INTO
module_log(gmt_create, gmt_modified, name, username, log)
VALUES
(now(), now(), ?, ?, "");
*/});
var INSERT_LOG_SQL = 'INSERT INTO module_log SET ?';
exports.create = function (data, callback) {
mysql.query(INSERT_LOG_SQL, [data.name, data.username], function (err, result) {
var now = new Date();
var args = {
gmt_create: now,
gmt_modified: now,
name: data.name,
username: data.username,
log: ''
}
mysql.query(INSERT_LOG_SQL, [args], function (err, result) {
if (err) {
return callback(err);
}
@@ -33,15 +35,7 @@ exports.create = function (data, callback) {
});
};
var APPEND_SQL = multiline(function () {/*
UPDATE
module_log
SET
log=CONCAT(log, ?),
gmt_modified=now()
WHERE
id=?;
*/});
var APPEND_SQL = 'UPDATE module_log SET log=CONCAT(log, ?), gmt_modified=now() WHERE id=?';
exports.append = function (id, log, callback) {
log = '\n' + log;
mysql.query(APPEND_SQL, [log, id], function (err) {
@@ -49,14 +43,7 @@ exports.append = function (id, log, callback) {
});
};
var SELECT_SQL = multiline(function () {/*
SELECT
*
FROM
module_log
WHERE
id=?;
*/});
var SELECT_SQL = 'SELECT * FROM module_log WHERE id=?';
exports.get = function (id, callback) {
mysql.queryOne(SELECT_SQL, [id], callback);
};

View File

@@ -0,0 +1,81 @@
/**!
* cnpmjs.org - proxy/module_maintainer.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:proxy:module_maintainer');
var mysql = require('../common/mysql');
var Module = require('./module');
var GET_MAINTANINERS_SQL = 'SELECT user FROM module_maintainer WHERE name = ?;';
exports.get = function* (name) {
var users = yield mysql.query(GET_MAINTANINERS_SQL, [name]);
return users.map(function (row) {
return row.user;
});
};
var ADD_SQL = 'INSERT INTO module_maintainer(name, user, gmt_create) \
VALUES (?, ?, now());';
function* add(name, username) {
try {
yield mysql.query(ADD_SQL, [name, username]);
} catch (err) {
if (err.code !== 'ER_DUP_ENTRY') {
throw err;
}
}
}
var REMOVE_SQL = 'DELETE FROM module_maintainer WHERE name = ? AND user IN (?);';
function* remove(name, usernames) {
return yield mysql.query(REMOVE_SQL, [name, usernames]);
}
exports.update = function* (name, maintainers) {
// maintainers should be [name1, name2, ...] format
// find out the exists maintainers then remove the deletes and add the left
if (maintainers.length === 0) {
return {
add: [],
remove: []
};
}
var exists = yield* exports.get(name);
var addUsers = maintainers;
var removeUsers = [];
if (exists.length > 0) {
for (var i = 0; i < exists.length; i++) {
var username = exists[i];
if (addUsers.indexOf(username) === -1) {
removeUsers.push(username);
}
}
}
var tasks = [];
for (var i = 0; i < addUsers.length; i++) {
tasks.push(add(name, addUsers[i]));
}
yield tasks;
// make sure all add users success then remove users
if (removeUsers.length > 0) {
yield* remove(name, removeUsers);
}
debug('add %d users, remove %d users', addUsers.length, removeUsers.length);
return {
add: addUsers,
remove: removeUsers
};
};

View File

@@ -17,11 +17,11 @@
var mysql = require('../common/mysql');
var multiline = require('multiline');
var ADD_SQL = multiline(function () {/*
INSERT iNTO
module_star(name, user)
var ADD_SQL = multiline(function () {;/*
INSERT INTO
module_star(name, user, gmt_create)
VALUES
(?, ?);
(?, ?, now());
*/});
exports.add = function *add(name, user) {
try {
@@ -33,7 +33,7 @@ exports.add = function *add(name, user) {
}
};
var REMOVE_SQL = multiline(function () {/*
var REMOVE_SQL = multiline(function () {;/*
DELETE FROM
module_star
WHERE
@@ -43,7 +43,7 @@ exports.remove = function *(name, user) {
return yield mysql.query(REMOVE_SQL, [name, user]);
};
var LIST_USERS_SQL = multiline(function () {/*
var LIST_USERS_SQL = multiline(function () {;/*
SELECT
user
FROM
@@ -58,7 +58,7 @@ exports.listUsers = function *(name) {
});
};
var LIST_USER_MODULES_SQL = multiline(function () {/*
var LIST_USER_MODULES_SQL = multiline(function () {;/*
SELECT
name
FROM

View File

@@ -0,0 +1,43 @@
/**!
* cnpmjs.org - proxy/module_unpublished.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var multiline = require('multiline');
var mysql = require('../common/mysql');
var SAVE_SQL = multiline(function () {;/*
INSERT INTO
module_unpublished(gmt_create, gmt_modified, name, package)
VALUES
(now(), now(), ?, ?)
ON DUPLICATE KEY UPDATE
gmt_modified=now(),
name=VALUES(name),
package=VALUES(package);
*/});
exports.add = function* (name, pkg) {
return yield mysql.query(SAVE_SQL, [name, JSON.stringify(pkg)]);
};
var GET_SQL = 'SELECT gmt_modified, name, package FROM module_unpublished WHERE name=?;';
exports.get = function* (name) {
var row = yield mysql.queryOne(GET_SQL, [name]);
if (row) {
row.package = JSON.parse(row.package);
}
return row;
};

View File

@@ -19,17 +19,18 @@ var config = require('../config');
var USER_AGENT = 'cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
function *request(url, options) {
function* request(url, options) {
options = options || {};
options.dataType = options.dataType || 'json';
options.timeout = options.timeout || 120000;
options.headers = {
'user-agent': USER_AGENT
};
url = config.sourceNpmRegistry + url;
var registry = options.registry || config.sourceNpmRegistry;
url = registry + url;
var r;
try {
r = yield *urllib.request(url, options);
r = yield* urllib.request(url, options);
} catch (err) {
var statusCode = err.status || -1;
var data = err.data || '[empty]';
@@ -42,6 +43,8 @@ function *request(url, options) {
return r;
}
exports.request = request;
exports.getUser = function *(name) {
var url = '/-/user/org.couchdb.user:' + name;
var r = yield *request(url);
@@ -70,7 +73,8 @@ exports.getAllSince = function *(startkey) {
exports.getShort = function *() {
var r = yield *request('/-/short', {
timeout: 300000
timeout: 300000,
registry: 'http://r.cnpmjs.org', // registry.npmjs.org/-/short is 404 now.
});
return r.data;
};

View File

@@ -17,6 +17,7 @@
var co = require('co');
var gather = require('co-gather');
var defer = require('co-defer');
var thunkify = require('thunkify-wrap');
var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
var EventEmitter = require('events').EventEmitter;
@@ -27,6 +28,7 @@ var crypto = require('crypto');
var urllib = require('co-urllib');
var utility = require('utility');
var ms = require('ms');
var urlparse = require('url').parse;
var nfs = require('../common/nfs');
var npm = require('./npm');
var common = require('../lib/common');
@@ -36,22 +38,19 @@ var Log = require('./module_log');
var config = require('../config');
var ModuleStar = require('./module_star');
var User = require('./user');
var ModuleUnpublished = require('./module_unpublished');
var USER_AGENT = 'sync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
function SyncModuleWorker(options) {
EventEmitter.call(this);
this._logId = options.logId;
this.startName = options.name;
if (!Array.isArray(options.name)) {
options.name = [options.name];
}
this.names = options.name;
// for (var i = 0; i < this.names.length; i++) {
// // ensure package name is lower case
// this.names[i] = this.names[i].toLowerCase();
// }
this.names = options.name || [];
this.startName = this.names[0];
this.username = options.username;
this.concurrency = options.concurrency || 1;
@@ -124,53 +123,89 @@ SyncModuleWorker.prototype.add = function (name) {
this.log(' add dependencies: %s', name);
};
SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
if (success) {
this.pushSuccess(name);
} else {
this.pushFail(name);
}
delete this.syncingNames[name];
var that = this;
// relase the stack: https://github.com/cnpm/cnpmjs.org/issues/328
defer.setImmediate(function *() {
that.log('[c#%s] setImmediate after, %s done, start next...', concurrencyId, name);
yield *that.next(concurrencyId);
});
};
SyncModuleWorker.prototype.next = function *(concurrencyId) {
var name = this.names.shift();
if (!name) {
return process.nextTick(this.finish.bind(this));
return setImmediate(this.finish.bind(this));
}
var that = this;
that.syncingNames[name] = true;
var pkg;
var pkg = null;
var status = 0;
// get from npm
try {
pkg = yield npm.get(name);
var result = yield npm.request('/' + name);
pkg = result.data;
status = result.status;
} catch (err) {
// if 404
if (err.res && err.res.statusCode === 404) {
that.pushSuccess(name);
} else {
that.pushFail(name);
if (!err.res || err.res.statusCode !== 404) {
var errMessage = err.name + ': ' + err.message;
that.log('[c#%s] [error] [%s] get package error: %s, status: %s',
concurrencyId, name, errMessage, status);
yield *that._doneOne(concurrencyId, name, false);
return;
}
that.log('[error] [%s] get package error: %s', name, err.stack);
delete that.syncingNames[name];
yield *that.next(concurrencyId);
return;
}
var unpublishedInfo = null;
if (status === 404) {
// check if it's unpublished
if (pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
unpublishedInfo = pkg.time.unpublished;
} else {
pkg = null;
}
}
if (!pkg) {
that.log('[error] [%s] get package error: package not exist', name);
delete that.syncingNames[name];
yield that.next(concurrencyId);
that.log('[c#%s] [error] [%s] get package error: package not exists, status: %s',
concurrencyId, name, status);
yield* that._doneOne(concurrencyId, name, true);
return;
}
that.log('[c#%d] [%s] start...', concurrencyId, name);
that.log('[c#%d] [%s] pkg status: %d, start...', concurrencyId, name, status);
if (unpublishedInfo) {
try {
yield* that._unpublished(name, unpublishedInfo);
} catch (err) {
that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
yield* that._doneOne(concurrencyId, name, false);
return;
}
return yield* that._doneOne(concurrencyId, name, true);
}
var versions;
try {
versions = yield that._sync(name, pkg);
versions = yield* that._sync(name, pkg);
} catch (err) {
that.pushFail(name);
that.log('[error] [%s] sync error: %s', name, err.stack);
delete that.syncingNames[name];
yield *that.next(concurrencyId);
that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
yield* that._doneOne(concurrencyId, name, false);
return;
}
that.log('[%s] synced success, %d versions: %s',
name, versions.length, versions.join(', '));
that.pushSuccess(name);
delete that.syncingNames[name];
yield that.next(concurrencyId);
that.log('[c#%d] [%s] synced success, %d versions: %s',
concurrencyId, name, versions.length, versions.join(', '));
yield* that._doneOne(concurrencyId, name, true);
};
function *_listStarUsers(modName) {
@@ -194,7 +229,61 @@ function *_saveNpmUser(username) {
yield User.saveNpmUser(user);
}
SyncModuleWorker.prototype._sync = function *(name, pkg) {
SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
var mods = yield Module.listByName(name);
this.log(' [%s] start unpublished %d versions from local cnpm registry',
name, mods.length);
if (this._isLocalModule(mods)) {
// publish on cnpm, dont sync this version package
this.log(' [%s] publish on local cnpm registry, don\'t sync', name);
return [];
}
var r = yield* ModuleUnpublished.add(name, unpublishedInfo);
this.log(' [%s] save unpublished info: %j to row#%s',
name, unpublishedInfo, r.insertId);
if (mods.length === 0) {
return;
}
yield [Module.removeByName(name), Module.removeTags(name)];
var keys = [];
for (var i = 0; i < mods.length; i++) {
var row = mods[i];
var dist = row.package.dist;
var key = dist.key;
if (!key) {
key = urlparse(dist.tarball).pathname;
}
key && keys.push(key);
}
if (keys.length === 0) {
return;
}
try {
yield keys.map(function (key) {
return nfs.remove(key);
});
} catch (err) {
// ignore error here
this.log(' [%s] delete nfs files: %j error: %s: %s',
name, keys, err.name, err.message);
}
this.log(' [%s] delete nfs files: %j success', name, keys);
};
SyncModuleWorker.prototype._isLocalModule = function (mods) {
for (var i = 0; i < mods.length; i++) {
var r = mods[i];
if (r.package && r.package._publish_on_cnpm) {
return true;
}
}
return false;
};
SyncModuleWorker.prototype._sync = function* (name, pkg) {
var username = this.username;
var that = this;
@@ -208,8 +297,15 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
var tagRows = result[1];
var existsStarUsers = result[2];
if (that._isLocalModule(moduleRows)) {
// publish on cnpm, dont sync this version package
that.log(' [%s] publish on local cnpm registry, don\'t sync', name);
return [];
}
hasModules = moduleRows.length > 0;
var map = {};
var localVersionNames = [];
for (var i = 0; i < moduleRows.length; i++) {
var r = moduleRows[i];
if (!r.package || !r.package.dist) {
@@ -217,12 +313,6 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
continue;
}
if (r.package && r.package._publish_on_cnpm) {
// publish on cnpm, dont sync this version package
that.log(' [%s] publish on local cnpm, don\'t sync', name);
return [];
}
if (r.version === 'next') {
continue;
}
@@ -230,6 +320,7 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
map.latest = r;
}
map[r.version] = r;
localVersionNames.push(r.version);
}
var tags = {};
@@ -255,11 +346,15 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
var maintainers = p.maintainers || [];
if (maintainers && !Array.isArray(maintainers)) {
// http://r.cnpmjs.org/jasmine-node
// TODO: "maintainers": "Martin H膫陇ger <martin.haeger@gmail.com>",
maintainers = [maintainers];
}
maintainers.forEach(function (m) {
npmUsernames[m.name.toLowerCase()] = 1;
if (m.name) {
npmUsernames[m.name.toLowerCase()] = 1;
}
});
}
@@ -275,11 +370,8 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
var times = pkg.time || {};
pkg.versions = pkg.versions || {};
var versionNames = Object.keys(times);
if (versionNames.length === 0) {
versionNames = Object.keys(pkg.versions);
}
if (versionNames.length === 0) {
var remoteVersionNames = Object.keys(pkg.versions);
if (remoteVersionNames.length === 0) {
that.log(' [%s] no times and no versions, hasModules: %s', name, hasModules);
if (!hasModules) {
// save a next module
@@ -313,9 +405,12 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
}
}
var versions = [];
for (var i = 0; i < versionNames.length; i++) {
var v = versionNames[i];
var remoteVersionNameMap = {};
// find out missing versions
for (var i = 0; i < remoteVersionNames.length; i++) {
var v = remoteVersionNames[i];
remoteVersionNameMap[v] = v;
var exists = map[v] || {};
var version = pkg.versions[v];
if (!version || !version.dist || !version.dist.tarball) {
@@ -360,9 +455,19 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
continue;
}
}
versions.push(version);
missingVersions.push(version);
}
// find out deleted versions
var deletedVersionNames = [];
for (var i = 0; i < localVersionNames.length; i++) {
var v = localVersionNames[i];
if (!remoteVersionNameMap[v]) {
deletedVersionNames.push(v);
}
}
// find out missing tags
var sourceTags = pkg['dist-tags'] || {};
for (var t in sourceTags) {
var sourceTagVersion = sourceTags[t];
@@ -370,18 +475,25 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
missingTags.push([t, sourceTagVersion]);
}
}
if (versions.length === 0) {
that.log(' [%s] all versions are exists', name);
} else {
versions.sort(function (a, b) {
return a.publish_time - b.publish_time;
});
that.log(' [%s] %d versions need to sync', name, versions.length);
// find out deleted tags
var deletedTags = [];
for (var t in tags) {
if (!sourceTags[t]) {
// not in remote tags, delete it from local registry
deletedTags.push(t);
}
}
missingVersions = versions;
var versionNames = [];
if (missingVersions.length === 0) {
that.log(' [%s] all versions are exists', name);
} else {
missingVersions.sort(function (a, b) {
return a.publish_time - b.publish_time;
});
that.log(' [%s] %d versions need to sync', name, missingVersions.length);
}
var syncedVersionNames = [];
var syncIndex = 0;
// sync missing versions
@@ -393,13 +505,27 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
}
try {
var result = yield that._syncOneVersion(index, syncModule);
versionNames.push(syncModule.version);
syncedVersionNames.push(syncModule.version);
} catch (err) {
that.log(' [%s:%d] error, version: %s, %s: %s',
that.log(' [%s:%d] sync error, version: %s, %s: %s',
syncModule.name, index, syncModule.version, err.name, err.message);
}
}
if (deletedVersionNames.length === 0) {
that.log(' [%s] no versions need to deleted', name);
} else {
that.log(' [%s] %d versions: %j need to deleted',
name, deletedVersionNames.length, deletedVersionNames);
try {
yield Module.removeByNameAndVersions(name, deletedVersionNames);
} catch (err) {
that.log(' [%s] delete error, %s: %s', name, err.name, err.message);
}
}
// sync missing descriptions
function *syncDes() {
if (missingDescriptions.length === 0) {
@@ -424,7 +550,13 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
}
// sync missing tags
function *syncTag() {
function* syncTag() {
if (deletedTags.length > 0) {
yield* Module.removeTagsByNames(name, deletedTags);
that.log(' [%s] deleted %d tags: %j',
name, deletedTags.length, deletedTags);
}
if (missingTags.length === 0) {
return;
}
@@ -527,7 +659,7 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
}
yield [syncDes(), syncTag(), syncReadme(), syncMissingStarUsers(), syncMissingUsers()];
return versionNames;
return syncedVersionNames;
};
SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
@@ -590,12 +722,14 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
// get tarball
var r = yield *urllib.request(downurl, options);
var statusCode = r.status || -1;
if (statusCode === 404) {
shasum = sourcePackage.dist.shasum;
return yield afterUpload({
url: downurl
});
}
// https://github.com/cnpm/cnpmjs.org/issues/325
// if (statusCode === 404) {
// shasum = sourcePackage.dist.shasum;
// return yield *afterUpload({
// url: downurl
// });
// }
if (statusCode !== 200) {
var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
err.name = 'DownloadTarballError';
@@ -612,6 +746,13 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
var end = thunkify.event(rs);
yield end(); // after end event emit
if (dataSize === 0) {
var err = new Error('Download ' + downurl + ' file size is zero');
err.name = 'DownloadTarballSizeZeroError';
err.data = sourcePackage;
throw err;
}
// check shasum
shasum = shasum.digest('hex');
if (shasum !== sourcePackage.dist.shasum) {
@@ -629,7 +770,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
};
// upload to NFS
var result = yield nfs.upload(filepath, options);
return yield afterUpload(result);
return yield *afterUpload(result);
} finally {
// remove tmp file whatever
fs.unlink(filepath, utility.noop);
@@ -651,6 +792,8 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
publish_time: sourcePackage.publish_time,
};
// delete _publish_on_cnpm, because other cnpm maybe sync from current cnpm
delete mod.package._publish_on_cnpm;
if (that._publish) {
// sync as publish
mod.package._publish_on_cnpm = true;
@@ -672,27 +815,35 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
mod.package.dist = dist;
var r = yield Module.add(mod);
that.log(' [%s:%s] done, insertId: %s, author: %s, version: %s, ' +
'size: %d, publish_time: %j, publish on cnpm: %s',
sourcePackage.name, versionIndex,
r.id,
author, mod.version, dataSize,
new Date(mod.publish_time),
that._publish);
that.log(' [%s:%s] done, insertId: %s, author: %s, version: %s, '
+ 'size: %d, publish_time: %j, publish on cnpm: %s',
sourcePackage.name, versionIndex,
r.id,
author, mod.version, dataSize,
new Date(mod.publish_time),
that._publish);
return r;
}
};
SyncModuleWorker.sync = function *(name, username, options) {
SyncModuleWorker.sync = function* (name, username, options) {
options = options || {};
var pkg = yield npm.get(name);
if (!pkg || !pkg._rev) {
var result = yield npm.request('/' + name);
var pkg = result.data;
if (result.status === 404 &&
(!pkg.time || !pkg.time.unpublished || !pkg.time.unpublished.time)) {
pkg = null;
}
if (!pkg || !pkg._id) {
return {
ok: false,
pkg: pkg
pkg: pkg,
statusCode: 404
};
}
var result = yield Log.create({name: name, username: username});
var worker = new SyncModuleWorker({
logId: result.id,

View File

@@ -20,7 +20,7 @@ var config = require('../config');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var DB_SIZE_SQL = multiline(function () {/*
var DB_SIZE_SQL = multiline(function () {;/*
SELECT
TABLE_NAME AS name, data_length, index_length
FROM
@@ -93,7 +93,7 @@ exports.get = function (callback) {
});
};
var PLUS_DELETE_MODULE_SQL = multiline(function () {/*
var PLUS_DELETE_MODULE_SQL = multiline(function () {;/*
UPDATE
total
SET
@@ -109,7 +109,7 @@ exports.getTotalInfo = function (callback) {
mysql.queryOne(TOTAL_INFO_SQL, callback);
};
var SET_LAST_SYNC_TIME_SQL = multiline(function () {/*
var SET_LAST_SYNC_TIME_SQL = multiline(function () {;/*
UPDATE
total
SET
@@ -121,7 +121,7 @@ exports.setLastSyncTime = function (time, callback) {
mysql.query(SET_LAST_SYNC_TIME_SQL, Number(time), callback);
};
var SET_LAST_EXIST_SYNC_TIME_SQL = multiline(function () {/*
var SET_LAST_EXIST_SYNC_TIME_SQL = multiline(function () {;/*
UPDATE
total
SET
@@ -133,7 +133,7 @@ exports.setLastExistSyncTime = function (time, callback) {
mysql.query(SET_LAST_EXIST_SYNC_TIME_SQL, Number(time), callback);
};
var UPDATE_SYNC_STATUS_SQL = multiline(function () {/*
var UPDATE_SYNC_STATUS_SQL = multiline(function () {;/*
UPDATE
total
SET
@@ -145,26 +145,25 @@ exports.updateSyncStatus = function (status, callback) {
mysql.query(UPDATE_SYNC_STATUS_SQL, [status], callback);
};
var UPDATE_SYNC_NUM_SQL = multiline(function () {/*
var UPDATE_SYNC_NUM_SQL = multiline(function () {;/*
UPDATE
total
SET
sync_status = ?,
need_sync_num = ?,
success_sync_num = ?,
fail_sync_num = ?,
left_sync_num = ?,
last_sync_module = ?
?
WHERE
name="total";
*/});
exports.updateSyncNum = function (params, callback) {
var query = [
params.syncStatus, params.need || 0,
params.success || 0, params.fail || 0, params.left || 0,
params.lastSyncModule,
];
mysql.query(UPDATE_SYNC_NUM_SQL, query, callback);
var arg = {
sync_status: params.syncStatus,
need_sync_num: params.need || 0,
success_sync_num: params.success || 0,
fail_sync_num: params.fail || 0,
left_sync_num: params.left || 0,
last_sync_module: params.lastSyncModule
};
mysql.query(UPDATE_SYNC_NUM_SQL, [arg], callback);
};
thunkify(exports);

View File

@@ -20,7 +20,7 @@ var config = require('../config');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var SELECT_USER_SQL = multiline(function () {/*
var SELECT_USER_SQL = multiline(function () {;/*
SELECT
id, rev, name, email, salt, password_sha, ip,
roles, json, npm_user, gmt_create, gmt_modified
@@ -66,13 +66,7 @@ exports.auth = function (name, password, callback) {
};
var INSERT_USER_SQL = multiline(function () {/*
INSERT INTO
user(rev, name, email, salt, password_sha,
ip, roles, gmt_create, gmt_modified)
VALUES
(?, ?, ?, ?, ?, ?, ?, now(), now());
*/});
var INSERT_USER_SQL = 'INSERT INTO user SET ?';
exports.add = function (user, callback) {
var roles = user.roles || [];
try {
@@ -81,23 +75,31 @@ exports.add = function (user, callback) {
roles = '[]';
}
var rev = '1-' + utility.md5(JSON.stringify(user));
var values = [rev, user.name, user.email, user.salt, user.password_sha, user.ip, roles];
mysql.query(INSERT_USER_SQL, values, function (err) {
var now = new Date();
var arg = {
rev: rev,
name: user.name,
email: user.email,
salt: user.salt,
password_sha: user.password_sha,
ip: user.ip,
roles: roles,
gmt_create: now,
gmt_modified: now
};
mysql.query(INSERT_USER_SQL, [arg], function (err) {
callback(err, {rev: rev});
});
};
var UPDATE_USER_SQL = multiline(function () {/*
var UPDATE_USER_SQL = multiline(function () {;/*
UPDATE
user
SET
rev=?,
email=?,
salt=?,
password_sha=?,
ip=?,
roles=?,
gmt_modified=now()
?
WHERE
name=? AND rev=?;
*/});
@@ -119,12 +121,21 @@ exports.update = function (user, callback) {
roles = '[]';
}
var values = [newRev, user.email, user.salt, user.password_sha, user.ip, roles, user.name, rev];
mysql.query(UPDATE_USER_SQL, values, function (err, data) {
if (err || !data.affectedRows) {
var arg = {
rev: newRev,
email: user.email,
salt: user.salt,
password_sha: user.password_sha,
ip: user.ip,
roles: roles,
gmt_modified: new Date()
};
mysql.query(UPDATE_USER_SQL, [arg, user.name, rev], function (err, data) {
if (err) {
return callback(err);
}
callback(null, {rev: newRev});
callback(null, {rev: newRev, result: data});
});
};
@@ -145,7 +156,7 @@ exports.saveNpmUser = function *(user) {
}
};
var LIST_BY_NAMES_SQL = multiline(function () {/*
var LIST_BY_NAMES_SQL = multiline(function () {;/*
SELECT
id, name, email, json
FROM

View File

@@ -15,7 +15,6 @@
* Module dependencies.
*/
var middlewares = require('koa-middlewares');
var limit = require('../middleware/limit');
var login = require('../middleware/login');
var publishable = require('../middleware/publishable');
@@ -26,7 +25,15 @@ var user = require('../controllers/registry/user');
var sync = require('../controllers/sync');
function routes(app) {
app.get('/', middlewares.jsonp(), total.show);
function* jsonp(next) {
yield* next;
if (this.body) {
this.jsonp = this.body;
}
}
app.get('/', jsonp, total.show);
//before /:name/:version
//get all modules, for npm search
@@ -36,27 +43,28 @@ function routes(app) {
app.get('/-/short', mod.listAllModuleNames);
// module
// scope package: params: [$name]
app.get(/\/(@[\w\-\.]+\/[\w\-\.]+)$/, syncByInstall, mod.show);
// scope package: params: [$name, $version]
app.get(/\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\.\-]+)$/, syncByInstall, mod.get);
app.get('/:name', syncByInstall, mod.show);
app.get('/:name/:version', syncByInstall, mod.get);
// try to add module
app.put('/:name', login, publishable, mod.add);
app.put('/:name', login, publishable, mod.addPackageAndDist);
// sync from source npm
app.put('/:name/sync', sync.sync);
app.get('/:name/sync/log/:id', sync.getSyncLog);
app.put('/:name/:tag', login, mod.updateTag);
// need limit by ip
app.get('/:name/download/:filename', limit, mod.download);
// put tarball
// https://registry.npmjs.org/cnpmjs.org/-/cnpmjs.org-0.0.0.tgz/-rev/1-c85bc65e8d2470cc4d82b8f40da65b8e
app.put('/:name/-/:filename/-rev/:rev', login, publishable, mod.upload);
// delete tarball
app.delete('/:name/download/:filename/-rev/:rev', login, publishable, mod.removeTar);
// put package.json to module
app.put('/:name/:version/-tag/latest', login, publishable, mod.updateLatest);
// update module, unpublish will PUT this
app.put('/:name/-rev/:rev', login, publishable, mod.updateOrRemove);
app.delete('/:name/-rev/:rev', login, publishable, mod.removeAll);

View File

@@ -23,20 +23,34 @@ var dist = require('../controllers/web/dist');
function routes(app) {
app.get('/total', total.show);
// scope package without version
app.get(/\/package\/(@[\w\-\.]+\/[\w\-\.]+)$/, pkg.display);
// scope package with version
app.get(/\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\d\.]+)$/, pkg.display);
app.get('/package/:name', pkg.display);
app.get('/package/:name/:version', pkg.display);
app.get(/\/browse\/keyword\/(@[\w\-\.]+\/[\w\-\.]+)$/, pkg.search);
app.get('/browse/keyword/:word', pkg.search);
app.get('/~:name', user.display);
app.get(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)$/, pkg.displaySync);
app.get('/sync/:name', pkg.displaySync);
app.put(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)$/, sync.sync);
app.put('/sync/:name', sync.sync);
// params: [$name, $id]
app.get(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)\/log\/(\d+)$/, sync.getSyncLog);
app.get('/sync/:name/log/:id', sync.getSyncLog);
app.get('/sync', pkg.displaySync);
app.get('/_list/search/search', pkg.rangeSearch);
app.get(/^\/dist(\/.+)?/, dist.redirect);
app.get(/^\/dist(\/.*)?/, dist.list);
}
module.exports = routes;

View File

@@ -18,29 +18,31 @@
var koa = require('koa');
var app = module.exports = koa();
var http = require('http');
var microtime = require('microtime');
var middlewares = require('koa-middlewares');
var routes = require('../routes/registry');
var logger = require('../common/logger');
var config = require('../config');
var session = require('../common/session');
var auth = require('../middleware/auth');
var staticCache = require('../middleware/static');
var notFound = require('../middleware/registry_not_found');
app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
app.use(middlewares.rewrite('/favicon.ico', '/public/favicon.ico'));
middlewares.jsonp(app);
app.use(middlewares.rt({headerName: 'X-ReadTime'}));
app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
app.use(staticCache);
app.keys = ['todokey', config.sessionSecret];
app.outputErrors = true;
app.proxy = true;
app.use(session);
app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
app.use(auth());
app.use(notFound);
app.use(middlewares.gzip());
app.use(middlewares.fresh());
if (config.enableCompress) {
app.use(middlewares.compress({threshold: 150}));
}
app.use(middlewares.conditional());
app.use(middlewares.etag());
/**

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - servers/web.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -18,13 +18,13 @@
var path = require('path');
var http = require('http');
var fs = require('fs');
var microtime = require('microtime');
var koa = require('koa');
var middlewares = require('koa-middlewares');
var markdown = require('koa-markdown');
var session = require('../common/session');
var opensearch = require('../middleware/opensearch');
var notFound = require('../middleware/web_not_found');
var staticCache = require('../middleware/static');
var auth = require('../middleware/auth');
var routes = require('../routes/web');
var logger = require('../common/logger');
@@ -34,23 +34,23 @@ var app = koa();
var rootdir = path.dirname(__dirname);
app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
app.use(middlewares.staticCache(path.join(__dirname, '..', 'public'), {
buffer: !config.debug,
maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
dir: path.join(rootdir, 'public')
}));
app.use(middlewares.rt({headerName: 'X-ReadTime'}));
app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
app.use(staticCache);
app.use(opensearch);
app.keys = ['todokey', config.sessionSecret];
app.outputErrors = true;
app.proxy = true;
app.use(session);
app.use(middlewares.bodyParser());
app.use(auth());
app.use(notFound);
app.use(middlewares.gzip());
app.use(middlewares.fresh());
if (config.enableCompress) {
app.use(middlewares.compress({threshold: 150}));
}
app.use(middlewares.conditional());
app.use(middlewares.etag());
var viewDir = path.join(rootdir, 'view', 'web');
@@ -63,20 +63,30 @@ var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
.replace('{{logoURL}}', config.logoURL);
fs.writeFileSync(layoutFile, layout);
// custom web readme home page support
var readmeFile = path.join(docDir, '_readme.md');
var readmeContent;
if (config.customReadmeFile) {
readmeContent = fs.readFileSync(config.customReadmeFile, 'utf8');
} else {
readmeContent = fs.readFileSync(path.join(docDir, 'readme.md'), 'utf8');
}
fs.writeFileSync(readmeFile, readmeContent);
app.use(markdown({
baseUrl: '/',
root: docDir,
layout: layoutFile,
titleHolder: '<%- locals.title %>',
bodyHolder: '<%- locals.body %>',
indexName: 'readme'
indexName: '_readme'
}));
var locals = {
config: config
};
middlewares.render(app, {
middlewares.ejs(app, {
root: viewDir,
viewExt: 'html',
layout: '_layout',

60
services/package.js Normal file
View File

@@ -0,0 +1,60 @@
/**!
* cnpmjs.org - services/package.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var Module = require('../proxy/module');
var ModuleMaintainer = require('../proxy/module_maintainer');
var User = require('../proxy/user');
exports.listMaintainers = function* (name) {
var names = yield* ModuleMaintainer.get(name);
if (names.length === 0) {
return names;
}
var users = yield* User.listByNames(names);
return users.map(function (user) {
return {
name: user.name,
email: user.email
};
});
};
exports.updateMaintainers = function* (name, usernames) {
var rs = yield [
ModuleMaintainer.update(name, usernames),
Module.updateLastModified(name),
];
return rs[0];
};
exports.isMaintainer = function* (name, username) {
var maintainers = yield* ModuleMaintainer.get(name);
if (maintainers.length === 0) {
// if not found maintainers, try to get from latest module package info
var latestMod = yield Module.getLatest(name);
var ms = latestMod && latestMod.package && latestMod.package.maintainers;
if (ms && ms.length > 0) {
maintainers = ms.map(function (user) {
return user.name;
});
}
}
if (maintainers.length === 0) {
// no maintainers, meaning this module is free for everyone
return true;
}
return maintainers.indexOf(username) >= 0;
};

View File

@@ -14,15 +14,16 @@
* Module dependencies.
*/
var config = require('../config');
var debug = require('debug')('cnpmjs.org:sync:index');
var co = require('co');
var ms = require('ms');
var mail = require('../common/mail');
var util = require('util');
var utility = require('utility');
var debug = require('debug')('cnpmjs.org:sync:index');
var config = require('../config');
var mail = require('../common/mail');
var Total = require('../proxy/total');
var logger = require('../common/logger');
var co = require('co');
var syncDistWorker = require('./sync_dist');
var sync = null;
@@ -60,11 +61,11 @@ var handleSync = co(function *() {
var data = yield *sync();
} catch (err) {
error = err;
error.message += ' (sync package error)';
logger.syncError(error);
}
if (config.debug) {
error && console.error(error.stack);
data && console.log(data);
} else {
data && logger.syncInfo(data);
if (!config.debug) {
sendMailToAdmin(error, data, new Date());
}
syncing = false;
@@ -73,7 +74,34 @@ var handleSync = co(function *() {
if (sync) {
handleSync();
setInterval(handleSync, ms('30m'));
setInterval(handleSync, ms(config.syncInterval));
}
var syncingDist = false;
var syncDist = co(function* syncDist() {
if (syncingDist) {
return;
}
syncingDist = true;
logger.syncInfo('Start syncing dist...');
try {
yield* syncDistWorker();
yield* syncDistWorker.syncPhantomjsDir();
} catch (err) {
err.message += ' (sync dist error)';
logger.syncError(err);
if (config.noticeSyncDistError) {
sendMailToAdmin(err, null, new Date());
}
}
syncingDist = false;
});
if (config.syncDist) {
syncDist();
setInterval(syncDist, ms(config.syncInterval));
} else {
logger.syncInfo('sync dist disable');
}
function sendMailToAdmin(err, result, syncTime) {
@@ -91,7 +119,7 @@ function sendMailToAdmin(err, result, syncTime) {
subject = 'Sync Error';
type = 'error';
html = util.format('Sync packages from official registry failed.\n' +
'Start sync time is %s.\nError message is %s.', syncTime, err.stack);
'Start sync time is %s.\nError message is %s: %s\n%s.', syncTime, err.name, err.message, err.stack);
} else if (result.fails && result.fails.length) {
subject = 'Sync Finished But Some Packages Failed';
type = 'warn';
@@ -99,7 +127,7 @@ function sendMailToAdmin(err, result, syncTime) {
'Start sync time is %s.\n %d packges sync failed: %j ...\n %d packages sync successes :%j ...',
syncTime, result.fails.length, result.fails.slice(0, 10),
result.successes.length, result.successes.slice(0, 10));
} else {
} else if (result.successes && result.successes.length) {
subject = 'Sync Finished';
type = 'log';
html = util.format('Sync packages from official registry finished.\n' +
@@ -107,10 +135,10 @@ function sendMailToAdmin(err, result, syncTime) {
syncTime, result.successes.length, result.successes.slice(0, 10));
}
debug('send email with type: %s, subject: %s, html: %s', type, subject, html);
if (type !== 'log') {
logger.syncInfo('send email with type: %s, subject: %s, html: %s', type, subject, html);
if (type && type !== 'log') {
mail[type](to, subject, html, function (err) {
if (err) {
logger.info('send email with type: %s, subject: %s, html: %s', type, subject, html);
logger.error(err);
}
});

View File

@@ -47,6 +47,13 @@ Status.prototype.start = function () {
this.timer = setInterval(this.log.bind(this), 30000);
};
Status.prototype.stop = function () {
this.log(true);
clearInterval(this.timer);
this.timer = null;
this.started = false;
};
Status.init = function (options, worker) {
var status = new Status(options);
status.start();
@@ -65,9 +72,7 @@ Status.init = function (options, worker) {
});
worker.on('end', function () {
status.started = false;
status.log(true);
clearInterval(status.timer);
status.stop();
});
};

View File

@@ -24,7 +24,6 @@ var Npm = require('../proxy/npm');
var Total = require('../proxy/total');
var SyncModuleWorker = require('../proxy/sync_module_worker');
var Module = require('../proxy/module');
var co = require('co');
var thunkify = require('thunkify-wrap');
function subtract(subtracter, minuend) {

340
sync/sync_dist.js Normal file
View File

@@ -0,0 +1,340 @@
/**!
* cnpmjs.org - sync/sync_dist.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:sync:sync_dist');
var fs = require('fs');
var urllib = require('co-urllib');
var co = require('co');
var bytes = require('bytes');
var crypto = require('crypto');
var utility = require('utility');
var thunkify = require('thunkify-wrap');
var cheerio = require('cheerio');
var urlResolve = require('url').resolve;
var common = require('../lib/common');
var Dist = require('../proxy/dist');
var config = require('../config');
var nfs = require('../common/nfs');
var logger = require('../common/logger');
var disturl = config.disturl;
var USER_AGENT = 'distsync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
module.exports = sync;
function* sync(name) {
name = name || '/';
yield* syncDir(name);
}
function* syncDir(fullname, info) {
var news = yield* sync.listdiff(fullname);
var files = [];
var dirs = [];
for (var i = 0; i < news.length; i++) {
var item = news[i];
if (item.type === 'dir') {
dirs.push(item);
} else {
files.push(item);
}
}
logger.syncInfo('sync remote:%s got %d new items, %d dirs, %d files to sync',
fullname, news.length, dirs.length, files.length);
for (var i = 0; i < files.length; i++) {
yield* syncFile(files[i]);
}
for (var i = 0; i < dirs.length; i++) {
var dir = dirs[i];
yield* syncDir(dir.parent + dir.name, dir);
}
if (info) {
logger.syncInfo('Save dir:%s %j to database', fullname, info);
yield* Dist.savedir(info);
}
logger.syncInfo('Sync %s finished, %d dirs, %d files',
fullname, dirs.length, files.length);
}
function* syncFile(info) {
var name = info.parent + info.name;
name = process.pid + name.replace(/\//g, '_'); // make sure no parent dir
var isPhantomjsURL = false;
var downurl = disturl + info.parent + info.name;
if (info.downloadURL) {
downurl = info.downloadURL;
isPhantomjsURL = true;
}
var filepath = common.getTarballFilepath(name);
var ws = fs.createWriteStream(filepath);
var options = {
writeStream: ws,
followRedirect: true,
timeout: 6000000, // 100 minutes download
headers: {
'user-agent': USER_AGENT
}
};
try {
logger.syncInfo('downloading %s %s to %s, isPhantomjsURL: %s',
bytes(info.size), downurl, filepath, isPhantomjsURL);
// get tarball
var r = yield *urllib.request(downurl, options);
var statusCode = r.status || -1;
logger.syncInfo('download %s got status %s, headers: %j',
downurl, statusCode, r.headers);
if (statusCode !== 200) {
var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
err.name = 'DownloadDistFileError';
throw err;
}
var shasum = crypto.createHash('sha1');
var dataSize = 0;
var rs = fs.createReadStream(filepath);
rs.on('data', function (data) {
shasum.update(data);
dataSize += data.length;
});
var end = thunkify.event(rs);
yield end(); // after end event emit
if (dataSize === 0) {
var err = new Error('Download ' + downurl + ' file size is zero');
err.name = 'DownloadDistFileZeroSizeError';
throw err;
}
if (isPhantomjsURL) {
debug('real size: %s, expect size: %s', dataSize, info.size);
if (dataSize < info.size) {
// phantomjs download page only show `6.7 MB`
var err = new Error('Download ' + downurl + ' file size is '
+ dataSize + ' not match ' + info.size);
err.name = 'DownloadDistFileSizeError';
throw err;
}
info.size = dataSize;
} else if (dataSize !== info.size) {
var err = new Error('Download ' + downurl + ' file size is '
+ dataSize + ' not match ' + info.size);
err.name = 'DownloadDistFileSizeError';
throw err;
}
shasum = shasum.digest('hex');
var args = {
key: '/dist' + info.parent + info.name,
size: info.size,
shasum: shasum,
};
// upload to NFS
logger.syncInfo('uploading %s to nfs:%s', filepath, args.key);
var result = yield nfs.upload(filepath, args);
info.url = result.url || result.key;
info.sha1 = shasum;
logger.syncInfo('upload %s to nfs:%s with size:%d, sha1:%s',
args.key, info.url, info.size, info.sha1);
} finally {
// remove tmp file whatever
fs.unlink(filepath, utility.noop);
}
logger.syncInfo('Sync dist file: %j done', info);
yield* Dist.savefile(info);
}
// <a href="latest/">latest/</a> 02-May-2014 14:45 -
// <a href="node-v0.4.10.tar.gz">node-v0.4.10.tar.gz</a> 26-Aug-2011 16:22 12410018
var FILE_RE = /^<a[^>]+>([^<]+)<\/a>\s+(\d+\-\w+\-\d+ \d+\:\d+)\s+([\-\d]+)/;
function* listdir(fullname) {
var url = disturl + fullname;
var result = yield* urllib.request(url, {
timeout: 60000,
});
debug('listdir %s got %s, %j', url, result.status, result.headers);
var html = result.data && result.data.toString() || '';
var lines = html.split('\n');
var items = [];
for (var i = 0; i < lines.length; i++) {
var m = FILE_RE.exec(lines[i].trim());
if (!m) {
continue;
}
var itemName = m[1].replace(/^\/+/, '');
if (!itemName) {
continue;
}
// filter /nightlies/*
if (itemName.indexOf('nightlies/') === 0) {
continue;
}
items.push({
name: itemName, // 'SHASUMS.txt', 'x64/'
date: m[2],
size: m[3] === '-' ? '-' : parseInt(m[3]),
type: m[3] === '-' ? 'dir' : 'file',
parent: fullname, // '/', '/v0.10.28/'
});
}
return items;
}
sync.listdiff = function* (fullname) {
var items = yield* listdir(fullname);
if (items.length === 0) {
return items;
}
var exists = yield* Dist.listdir(fullname);
debug('listdiff %s got %s exists items', fullname, exists.length);
var map = {};
for (var i = 0; i < exists.length; i++) {
var item = exists[i];
map[item.name] = item;
}
var news = [];
for (var i = 0; i < items.length; i++) {
var item = items[i];
var exist = map[item.name];
if (!exist || exist.date !== item.date) {
news.push(item);
continue;
}
if (item.size !== '-' && item.size !== exist.size) {
news.push(item);
continue;
}
debug('skip %s', item.name);
}
return news;
};
function* syncPhantomjsDir() {
var fullname = '/phantomjs/';
var files = yield* sync.listPhantomjsDiff(fullname);
logger.syncInfo('sync remote:%s got %d files to sync',
fullname, files.length);
for (var i = 0; i < files.length; i++) {
yield* syncFile(files[i]);
}
logger.syncInfo('SyncPhantomjsDir %s finished, %d files',
fullname, files.length);
}
sync.syncPhantomjsDir = syncPhantomjsDir;
// <tr class="iterable-item" id="download-301626">
// <td class="name"><a class="execute" href="/ariya/phantomjs/downloads/phantomjs-1.9.7-windows.zip">phantomjs-1.9.7-windows.zip</a></td>
// <td class="size">6.7 MB</td>
// <td class="uploaded-by"><a href="/Vitallium">Vitallium</a></td>
// <td class="count">122956</td>
// <td class="date">
// <div>
// <time datetime="2014-01-27T18:29:53.706942" data-title="true">2014-01-27</time>
// </div>
// </td>
// <td class="delete">
//
// </td>
// </tr>
function* listPhantomjsDir(fullname) {
var url = 'https://bitbucket.org/ariya/phantomjs/downloads';
var result = yield* urllib.request(url, {
timeout: 60000,
});
debug('listPhantomjsDir %s got %s, %j', url, result.status, result.headers);
var html = result.data && result.data.toString() || '';
var $ = cheerio.load(html);
var items = [];
$('tr.iterable-item').each(function (i, el) {
var $el = $(this);
var $link = $el.find('.name a');
var name = $link.text();
var downloadURL = $link.attr('href');
if (!name || !downloadURL || !/\.(zip|bz2|gz)$/.test(downloadURL)) {
return;
}
downloadURL = urlResolve(url, downloadURL);
var size = parseInt(bytes($el.find('.size').text().toLowerCase().replace(/\s/g, '')));
if (size > 1024 * 1024) {
size -= 1024 * 1024;
} else if (size > 1024) {
size -= 1024;
} else {
size -= 10;
}
var date = $el.find('.date time').text();
items.push({
name: name, // 'SHASUMS.txt', 'x64/'
date: date,
size: size,
type: 'file',
parent: fullname,
downloadURL: downloadURL,
});
});
return items;
}
sync.listPhantomjsDir = listPhantomjsDir;
sync.listPhantomjsDiff = function* (fullname) {
var items = yield* listPhantomjsDir(fullname);
if (items.length === 0) {
return items;
}
var exists = yield* Dist.listdir(fullname);
debug('listdiff %s got %s exists items', fullname, exists.length);
var map = {};
for (var i = 0; i < exists.length; i++) {
var item = exists[i];
map[item.name] = item;
}
var news = [];
for (var i = 0; i < items.length; i++) {
var item = items[i];
var exist = map[item.name];
if (!exist || exist.date !== item.date) {
news.push(item);
continue;
}
// if (item.size !== exist.size) {
// news.push(item);
// continue;
// }
debug('skip %s', item.name);
}
return news;
};

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,120 @@
/**!
* cnpmjs.org - test/controllers/registry/module/scope_package.test.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var request = require('supertest');
var app = require('../../../../servers/registry');
var utils = require('../../../utils');
describe('controllers/registry/module/scope_package.test.js', function () {
var pkgname = '@cnpm/test-scope-package';
var pkgURL = '/@' + encodeURIComponent(pkgname.substring(1));
before(function (done) {
// add scope package
var pkg = utils.getPackage(pkgname, '0.0.1', utils.admin);
request(app.listen())
.put(pkgURL)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(201, function (err) {
should.not.exist(err);
pkg = utils.getPackage(pkgname, '0.0.2', utils.admin);
// publish 0.0.2
request(app.listen())
.put(pkgURL)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(201, done);
});
});
it('should get scope package info: /@scope%2Fname', function (done) {
request(app.listen())
.get(pkgURL)
.expect(200, function (err, res) {
should.not.exist(err);
var pkg = res.body;
pkg.name.should.equal(pkgname);
pkg.versions.should.have.keys('0.0.1', '0.0.2');
pkg['dist-tags'].latest.should.equal('0.0.2');
pkg.versions['0.0.1'].name.should.equal(pkgname);
pkg.versions['0.0.1'].dist.tarball
.should.containEql('/@cnpm/test-scope-package/download/@cnpm/test-scope-package-0.0.1.tgz');
done();
});
});
it('should get scope package info: /@scope/name', function (done) {
request(app.listen())
.get('/' + pkgname)
.expect(200, function (err, res) {
should.not.exist(err);
var pkg = res.body;
pkg.name.should.equal(pkgname);
pkg.versions.should.have.keys('0.0.1', '0.0.2');
pkg['dist-tags'].latest.should.equal('0.0.2');
pkg.versions['0.0.1'].name.should.equal(pkgname);
pkg.versions['0.0.1'].dist.tarball
.should.containEql('/@cnpm/test-scope-package/download/@cnpm/test-scope-package-0.0.1.tgz');
done();
});
});
it('should get scope package info: /%40scope%2Fname', function (done) {
request(app.listen())
.get('/' + encodeURIComponent(pkgname))
.expect(200, function (err, res) {
should.not.exist(err);
var pkg = res.body;
pkg.name.should.equal(pkgname);
pkg.versions.should.have.keys('0.0.1', '0.0.2');
pkg['dist-tags'].latest.should.equal('0.0.2');
pkg.versions['0.0.1'].name.should.equal(pkgname);
pkg.versions['0.0.1'].dist.tarball
.should.containEql('/@cnpm/test-scope-package/download/@cnpm/test-scope-package-0.0.1.tgz');
done();
});
});
it('should get scope package with version', function (done) {
request(app.listen())
.get('/' + pkgname + '/0.0.1')
.expect(200, function (err, res) {
should.not.exist(err);
var pkg = res.body;
pkg.name.should.equal(pkgname);
pkg.version.should.equal('0.0.1');
pkg.dist.tarball
.should.containEql('/@cnpm/test-scope-package/download/@cnpm/test-scope-package-0.0.1.tgz');
done();
});
});
it('should get scope package with tag', function (done) {
request(app.listen())
.get('/' + pkgname + '/latest')
.expect(200, function (err, res) {
should.not.exist(err);
var pkg = res.body;
pkg.name.should.equal(pkgname);
pkg.version.should.equal('0.0.2');
pkg.dist.tarball
.should.containEql('/@cnpm/test-scope-package/download/@cnpm/test-scope-package-0.0.2.tgz');
done();
});
});
});

View File

@@ -35,17 +35,17 @@ describe('controllers/registry/user.test.js', function () {
describe('GET /-/user/org.couchdb.user:name', function () {
it('should return user info', function (done) {
request(app)
.get('/-/user/org.couchdb.user:cnpmjstest1')
.get('/-/user/org.couchdb.user:cnpmjstest10')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type',
'_cnpm_meta', 'roles', 'date');
res.body.name.should.equal('cnpmjstest1');
res.body.name.should.equal('cnpmjstest10');
done();
});
});
it('should return npm user info', function (done) {
it.skip('should return npm user info', function (done) {
request(app)
.get('/-/user/org.couchdb.user:fengmk2')
.expect(200, function (err, res) {
@@ -167,7 +167,7 @@ describe('controllers/registry/user.test.js', function () {
}, function (err, res) {
should.not.exist(err);
should.exist(res.headers['set-cookie']);
res.headers['set-cookie'].join(';').should.include('AuthSession=');
res.headers['set-cookie'].join(';').should.containEql('AuthSession=');
done();
});
});
@@ -200,7 +200,7 @@ describe('controllers/registry/user.test.js', function () {
.expect(500, done);
});
it('should 409 when req.body.rev error', function (done) {
it('should 201 when req.body.rev error', function (done) {
request(app)
.put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
.set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
@@ -210,7 +210,7 @@ describe('controllers/registry/user.test.js', function () {
email: 'cnpmjstest10@cnpmjs.org',
rev: '1-123'
})
.expect(409, done);
.expect(201, done);
});
it('should 201 update ok', function (done) {

View File

@@ -27,8 +27,8 @@ var webApp = require('../../servers/web');
describe('controllers/sync.test.js', function () {
before(function (done) {
done = pedding(2, done);
registryApp.listen(0, done);
webApp.listen(0, done);
registryApp = registryApp.listen(0, done);
webApp = webApp.listen(0, done);
});
afterEach(mm.restore);
@@ -116,4 +116,16 @@ describe('controllers/sync.test.js', function () {
});
});
});
describe('scope package', function () {
it('should sync scope package not found', function (done) {
request(webApp)
.put('/sync/@cnpm/not-exists-package')
.expect({
"ok":false,
"reason":"can not found @cnpm/not-exists-package in the source registry"
})
.expect(404, done);
});
});
});

View File

@@ -16,22 +16,126 @@
var should = require('should');
var request = require('supertest');
var pedding = require('pedding');
var mm = require('mm');
var app = require('../../../servers/web');
var Dist = require('../../../proxy/dist');
describe('controllers/web/dist.test.js', function () {
before(function (done) {
app.listen(0, done);
});
after(function (done) {
app.close(done);
});
describe('GET /dist', function (done) {
it('should 302 to config.disturl', function (done) {
afterEach(mm.restore);
describe('GET /dist/*', function (done) {
it('should GET /dist redirect to /dist/', function (done) {
request(app)
.get('/dist')
.expect('Location', 'http://dist.u.qiniudn.com/')
.expect(302, done);
.expect(302)
.expect('Location', '/dist/', done);
});
it('should GET /dist/ show file list', function (done) {
request(app)
.get('/dist/')
.expect('Content-Type', 'text/html; charset=utf-8')
.expect(200, function (err, res) {
should.not.exist(err);
res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/</title>');
done();
});
});
it('should mock return files and dirs', function (done) {
mm(Dist, 'listdir', function* () {
return [
{name: 'ooxx/', date: '02-May-2014 00:54'},
{name: 'foo.txt', size: 1024, date: '02-May-2014 00:54'},
];
});
request(app)
.get('/dist/v1.0.0/')
.expect('Content-Type', 'text/html; charset=utf-8')
.expect(200, function (err, res) {
should.not.exist(err);
res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/v1.0.0/</title>');
res.text.should.containEql('<h1>Mirror index of <a target="_blank" href="http://nodejs.org/dist/v1.0.0/">http://nodejs.org/dist/v1.0.0/</a></h1>');
res.text.should.containEql('<a href="ooxx/">ooxx/</a> 02-May-2014 00:54 -\n');
res.text.should.containEql('<a href="foo.txt">foo.txt</a> 02-May-2014 00:54 1024\n');
done();
});
});
it('should list files and dirs', function (done) {
mm(Dist, 'listdir', function* () {
return [
{name: 'npm/', date: '02-May-2014 00:54'},
{name: 'npm-versions.txt', size: 1676, date: '02-May-2014 00:54'},
];
});
request(app)
.get('/dist/')
.expect('Content-Type', 'text/html; charset=utf-8')
.expect(200, function (err, res) {
should.not.exist(err);
res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/</title>');
res.text.should.containEql('<h1>Mirror index of <a target="_blank" href="http://nodejs.org/dist/">http://nodejs.org/dist/</a></h1>');
res.text.should.containEql('<a href="npm/">npm/</a> 02-May-2014 00:54 -\n');
res.text.should.containEql('<a href="npm-versions.txt">npm-versions.txt</a> 02-May-2014 00:54 1676\n');
done();
});
});
});
describe('GET /dist/ files', function () {
it('should redirect to nfs url', function (done) {
mm(Dist, 'getfile', function* () {
return {
name: 'foo.txt', size: 1024, date: '02-May-2014 00:54',
url: 'http://mock.com/dist/v0.10.28/SHASUMS.txt'
};
});
request(app)
.get('/dist/v0.10.28/SHASUMS.txt')
.expect(302)
.expect('Location', 'http://mock.com/dist/v0.10.28/SHASUMS.txt', done);
});
it('should GET /dist/npm-versions.txt redirect to nfs url', function (done) {
mm(Dist, 'getfile', function* (fullname) {
fullname.should.equal('/npm-versions.txt');
return {
name: 'npm-versions.txt', size: 1024, date: '02-May-2014 00:54',
url: 'http://mock.com/dist/npm-versions.txt'
};
});
request(app)
.get('/dist/npm-versions.txt')
.expect(302)
.expect('Location', 'http://mock.com/dist/npm-versions.txt', done);
});
it('should download nfs file and send it', function (done) {
mm(Dist, 'getfile', function* () {
return {
name: 'foo.txt',
size: 1264,
date: '02-May-2014 00:54',
url: '/dist/v0.10.28/SHASUMS.txt'
};
});
request(app)
.get('/dist/v0.10.28/SHASUMS.txt')
.expect(200)
.expect(/6eff580cc8460741155d42ef1ef537961194443f/, done);
});
});
});

View File

@@ -17,14 +17,30 @@
var should = require('should');
var request = require('supertest');
var mm = require('mm');
var path = require('path');
var mysql = require('../../../common/mysql');
var app = require('../../../servers/web');
var registry = require('../../../servers/registry');
var pkg = require('../../../controllers/web/package');
var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
describe('controllers/web/package.test.js', function () {
var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
before(function (done) {
app.listen(0, done);
registry.listen(0, function () {
var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
request(registry)
.put('/' + pkg.name)
.set('authorization', baseauth)
.send(pkg)
.expect(201, function () {
app.listen(0, done);
});
});
});
after(function (done) {
app.close(done);
});
@@ -32,10 +48,10 @@ describe('controllers/web/package.test.js', function () {
afterEach(mm.restore);
describe('GET /_list/search/search', function () {
it('should search with "c"', function (done) {
it('should search with "m"', function (done) {
request(app)
.get('/_list/search/search?startkey="c"&limit=2')
.expect('content-type', 'application/json')
.get('/_list/search/search?startkey="m"&limit=2')
.expect('content-type', 'application/json; charset=utf-8')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('rows');
@@ -48,9 +64,9 @@ describe('controllers/web/package.test.js', function () {
});
});
it('should search with c', function (done) {
it('should search with m', function (done) {
request(app)
.get('/_list/search/search?startkey=c&limit=2')
.get('/_list/search/search?startkey=m&limit=2')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('rows');
@@ -77,16 +93,16 @@ describe('controllers/web/package.test.js', function () {
describe('GET /package/:name', function (done) {
it('should get 200', function (done) {
request(app)
.get('/package/cutter')
.get('/package/mk2testmodule')
.expect(200)
.expect('content-encoding', 'gzip')
// .expect('content-encoding', 'gzip')
.expect('content-type', 'text/html; charset=utf-8')
.expect(/<div id="package">/)
.expect(/<th>Maintainers<\/th>/)
.expect(/<th>Version<\/th>/, function (err, res) {
should.not.exist(err);
res.should.have.header('etag');
res.text.should.include('<meta charset="utf-8">');
res.text.should.containEql('<meta charset="utf-8">');
done();
});
});
@@ -101,7 +117,7 @@ describe('controllers/web/package.test.js', function () {
describe('GET /package/:name/:version', function (done) {
it('should 200 when get by version', function (done) {
request(app)
.get('/package/cutter/0.0.2')
.get('/package/mk2testmodule/0.0.1')
.expect(200)
.expect(/<div id="package">/)
.expect(/<th>Maintainers<\/th>/)
@@ -110,21 +126,22 @@ describe('controllers/web/package.test.js', function () {
it('should 200 when get by tag', function (done) {
request(app)
.get('/package/cutter/latest')
.get('/package/mk2testmodule/latest')
.expect(200)
.expect(/<div id="package">/)
.expect(/<th>Maintainers<\/th>/)
.expect(/<th>Version<\/th>/, done);
});
it('should 404 when get by version not exist', function (done) {
request(app)
.get('/package/cutter/1.1.2')
.get('/package/mk2testmodule/1.1.2')
.expect(404, done);
});
it('should 404 when get by tag', function (done) {
request(app)
.get('/package/cutter/notexisttag')
.get('/package/mk2testmodule/notexisttag')
.expect(404, done);
});
});
@@ -132,14 +149,14 @@ describe('controllers/web/package.test.js', function () {
describe('GET /browse/keyword/:word', function () {
it('should list by keyword ok', function (done) {
request(app)
.get('/browse/keyword/cnpm')
.get('/browse/keyword/mk2testmodule')
.expect(200)
.expect(/Packages match/, done);
});
it('should list by keyword with json ok', function (done) {
request(app)
.get('/browse/keyword/cnpm?type=json')
.get('/browse/keyword/mk2testmodule?type=json')
.expect(200)
.expect('content-type', 'application/json; charset=utf-8', done);
});

View File

@@ -0,0 +1,99 @@
/**!
* cnpmjs.org - test/controllers/web/package/scope_package.test.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var request = require('supertest');
var registry = require('../../../../servers/registry');
var web = require('../../../../servers/web');
var utils = require('../../../utils');
describe('controllers/web/package/scope_package.test.js', function () {
var pkgname = '@cnpm/test-web-scope-package';
var pkgURL = '/@' + encodeURIComponent(pkgname.substring(1));
before(function (done) {
// add scope package
var pkg = utils.getPackage(pkgname, '0.0.1', utils.admin);
request(registry.listen())
.put(pkgURL)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(201, function (err) {
should.not.exist(err);
pkg = utils.getPackage(pkgname, '0.0.2', utils.admin);
// publish 0.0.2
request(registry.listen())
.put(pkgURL)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(201, done);
});
});
it('should show scope package info page: /@scope%2Fname', function (done) {
request(web.listen())
.get('/package' + pkgURL)
.expect(200, function (err, res) {
should.not.exist(err);
var body = res.text;
body.should.containEql('$ cnpm install @cnpm/test-web-scope-package');
body.should.containEql('/@cnpm/test-web-scope-package/download/@cnpm/test-web-scope-package-0.0.2.tgz');
done();
});
});
it('should show scope package info page: encodeURIComponent("/@scope/name")', function (done) {
request(web.listen())
.get('/package/' + encodeURIComponent(pkgname))
.expect(200, function (err, res) {
should.not.exist(err);
var body = res.text;
body.should.containEql('$ cnpm install @cnpm/test-web-scope-package');
body.should.containEql('/@cnpm/test-web-scope-package/download/@cnpm/test-web-scope-package-0.0.2.tgz');
done();
});
});
it('should show scope package info page: /@scope/name', function (done) {
request(web.listen())
.get('/package/' + pkgname)
.expect(200, function (err, res) {
should.not.exist(err);
var body = res.text;
body.should.containEql('$ cnpm install @cnpm/test-web-scope-package');
body.should.containEql('/@cnpm/test-web-scope-package/download/@cnpm/test-web-scope-package-0.0.2.tgz');
done();
});
});
it('should /@scope/name/ 404', function (done) {
request(web.listen())
.get('/package/' + pkgname + '/')
.expect(404, done);
});
it('should show scope package with version: /@scope/name/0.0.2', function (done) {
request(web.listen())
.get('/package/' + pkgname + '/0.0.2')
.expect(200, function (err, res) {
should.not.exist(err);
var body = res.text;
body.should.containEql('$ cnpm install @cnpm/test-web-scope-package');
body.should.containEql('/@cnpm/test-web-scope-package/download/@cnpm/test-web-scope-package-0.0.2.tgz');
done();
});
});
});

View File

@@ -29,7 +29,7 @@ describe('controllers/web/user.test.js', function () {
describe('GET /~:name', function (done) {
it('should get 200', function (done) {
request(app)
.get('/~dead_horse')
.get('/~cnpmjstest10')
.expect(200)
.expect('content-type', 'text/html; charset=utf-8')
.expect(/<div id="profile">/)
@@ -44,7 +44,7 @@ describe('controllers/web/user.test.js', function () {
it('should get not eixst user but have modules 200', function (done) {
request(app)
.get('/~tjholowaychuk')
.get('/~cnpmjstest101')
.expect(200)
.expect(/<div id="profile">/)
.expect(/Packages by/, done);

1
test/fixtures/cnpmtest-package.json vendored Normal file
View File

@@ -0,0 +1 @@
{"_id":"cnpmtest-package","_rev":"382002","name":"cnpmtest-package","description":"cnpmtest-package","dist-tags":{"latest":"0.0.0"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"time":{"modified":"2014-04-09T10:54:45.000Z","created":"2014-04-09T10:54:39.425Z","0.0.0":"2014-04-09T10:54:39.425Z"},"users":{},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"versions":{"0.0.0":{"name":"cnpmtest-package","version":"0.0.0","description":"cnpmtest-package","main":"index.js","scripts":{"test":"make test-all"},"config":{"cov":{"threshold":100}},"dependencies":{},"devDependencies":{"autod":"*","contributors":"*","should":"*","jshint":"*","cov":"*","istanbul-harmony":"*","mocha":"*"},"homepage":"https://github.com/{{group}}/cnpmtest-package","repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"keywords":["cnpmtest-package"],"engines":{"node":">= 0.10.0"},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"license":"MIT","contributors":[],"_id":"cnpmtest-package@0.0.0","dist":{"shasum":"81ea57c24a7f9f6e7263385116fc433f0ad9e179","size":1839,"noattachment":false,"tarball":"http://r.cnpmjs.org/cnpmtest-package/download/cnpmtest-package-0.0.0.tgz"},"_from":".","_npmVersion":"1.4.6","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"directories":{},"publish_time":1397040879425,"_cnpm_publish_time":1397040879425}},"readme":"cnpmtest-package\n=======\n\n[![Build Status](https://secure.travis-ci.org/{{group}}/cnpmtest-package.png)](http://travis-ci.org/{{group}}/cnpmtest-package)\n[![Dependency Status](https://gemnasium.com/{{group}}/cnpmtest-package.png)](https://gemnasium.com/{{group}}/cnpmtest-package)\n\n[![NPM](https://nodei.co/npm/cnpmtest-package.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmtest-package/)\n\n![logo](https://raw.github.com/{{group}}/cnpmtest-package/master/logo.png)\n\ncnpmtest-package desc\n\n## Install\n\n```bash\n$ npm install cnpmtest-package\n```\n\n## Usage\n\n```js\nvar cnpmtest-package = require('cnpmtest-package');\n\ncnpmtest-package.foo(function (err) {\n\n});\n```\n\n## License\n\n(The MIT License)\n\nCopyright (c) 2014 fengmk2 &lt;fengmk2@gmail.com&gt; and other contributors\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n'Software'), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n","_attachments":{},"homepage":"https://github.com/{{group}}/cnpmtest-package","bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"license":"MIT"}

16
test/fixtures/scope-package/.gitignore vendored Normal file
View File

@@ -0,0 +1,16 @@
coverage.html
*.seed
*.log
*.csv
*.dat
*.out
*.pid
*.gz
pids
logs
results
node_modules
npm-debug.log
coverage/

View File

@@ -0,0 +1,4 @@
node_modules/
coverage/
.tmp/
.git/

95
test/fixtures/scope-package/.jshintrc vendored Normal file
View File

@@ -0,0 +1,95 @@
{
// JSHint Default Configuration File (as on JSHint website)
// See http://jshint.com/docs/ for more details
"maxerr" : 50, // {int} Maximum error before stopping
// Enforcing
"bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
"camelcase" : false, // true: Identifiers must be in camelCase
"curly" : true, // true: Require {} for every new block or scope
"eqeqeq" : true, // true: Require triple equals (===) for comparison
"forin" : false, // true: Require filtering for..in loops with obj.hasOwnProperty()
"immed" : false, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
"indent" : false, // {int} Number of spaces to use for indentation
"latedef" : false, // true: Require variables/functions to be defined before being used
"newcap" : false, // true: Require capitalization of all constructor functions e.g. `new F()`
"noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee`
"noempty" : true, // true: Prohibit use of empty blocks
"nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment)
"plusplus" : false, // true: Prohibit use of `++` & `--`
"quotmark" : false, // Quotation mark consistency:
// false : do nothing (default)
// true : ensure whatever is used is consistent
// "single" : require single quotes
// "double" : require double quotes
"undef" : true, // true: Require all non-global variables to be declared (prevents global leaks)
"unused" : false, // true: Require all defined variables be used
"strict" : true, // true: Requires all functions run in ES5 Strict Mode
"trailing" : false, // true: Prohibit trailing whitespaces
"maxparams" : false, // {int} Max number of formal params allowed per function
"maxdepth" : false, // {int} Max depth of nested blocks (within functions)
"maxstatements" : false, // {int} Max number statements per function
"maxcomplexity" : false, // {int} Max cyclomatic complexity per function
"maxlen" : false, // {int} Max number of characters per line
// Relaxing
"asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons)
"boss" : true, // true: Tolerate assignments where comparisons would be expected
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
"eqnull" : false, // true: Tolerate use of `== null`
"es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
"esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`)
"moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// (ex: `for each`, multiple try/catch, function expression…)
"evil" : false, // true: Tolerate use of `eval` and `new Function()`
"expr" : true, // true: Tolerate `ExpressionStatement` as Programs
"funcscope" : false, // true: Tolerate defining variables inside control statements"
"globalstrict" : false, // true: Allow global "use strict" (also enables 'strict')
"iterator" : false, // true: Tolerate using the `__iterator__` property
"lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block
"laxbreak" : true, // true: Tolerate possibly unsafe line breakings
"laxcomma" : false, // true: Tolerate comma-first style coding
"loopfunc" : false, // true: Tolerate functions being defined in loops
"multistr" : true, // true: Tolerate multi-line strings
"proto" : false, // true: Tolerate using the `__proto__` property
"scripturl" : false, // true: Tolerate script-targeted URLs
"smarttabs" : false, // true: Tolerate mixed tabs/spaces when used for alignment
"shadow" : true, // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
"sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation
"supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;`
"validthis" : false, // true: Tolerate using this in a non-constructor function
// Environments
"browser" : true, // Web Browser (window, document, etc)
"couch" : false, // CouchDB
"devel" : true, // Development/debugging (alert, confirm, etc)
"dojo" : false, // Dojo Toolkit
"jquery" : false, // jQuery
"mootools" : false, // MooTools
"node" : true, // Node.js
"nonstandard" : false, // Widely adopted globals (escape, unescape, etc)
"prototypejs" : false, // Prototype and Scriptaculous
"rhino" : false, // Rhino
"worker" : false, // Web Workers
"wsh" : false, // Windows Scripting Host
"yui" : false, // Yahoo User Interface
"noyield" : true, // allow generators without a yield
// Legacy
"nomen" : false, // true: Prohibit dangling `_` in variables
"onevar" : false, // true: Allow only one `var` statement per function
"passfail" : false, // true: Stop on first error
"white" : false, // true: Check against strict whitespace and indentation rules
// Custom Globals
"globals" : { // additional predefined global variables
// mocha
"describe": true,
"it": true,
"before": true,
"afterEach": true,
"beforeEach": true,
"after": true
}
}

View File

@@ -0,0 +1,8 @@
test/
Makefile
.travis.yml
logo.png
.jshintignore
.jshintrc
.gitingore
coverage/

View File

@@ -0,0 +1,6 @@
language: node_js
node_js:
- '0.11'
- '0.10'
script: "make test-travis"
after_script: "npm install coveralls@2 && cat ./coverage/lcov.info | coveralls"

0
test/fixtures/scope-package/AUTHORS vendored Normal file
View File

21
test/fixtures/scope-package/LICENSE.txt vendored Normal file
View File

@@ -0,0 +1,21 @@
This software is licensed under the MIT License.
Copyright (c) 2014 fengmk2 <fengmk2@gmail.com> and other contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

51
test/fixtures/scope-package/Makefile vendored Normal file
View File

@@ -0,0 +1,51 @@
TESTS = test/*.test.js
REPORTER = spec
TIMEOUT = 1000
MOCHA_OPTS =
install:
@npm install --registry=https://registry.npm.taobao.org --disturl=https://npm.taobao.org/dist
jshint: install
@./node_modules/.bin/jshint .
test: install
@NODE_ENV=test ./node_modules/.bin/mocha \
--harmony \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
$(MOCHA_OPTS) \
$(TESTS)
test-cov cov: install
@NODE_ENV=test node --harmony \
node_modules/.bin/istanbul cover --preserve-comments \
./node_modules/.bin/_mocha \
-- \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
$(MOCHA_OPTS) \
$(TESTS)
@./node_modules/.bin/cov coverage
test-travis: install
@NODE_ENV=test node --harmony \
node_modules/.bin/istanbul cover --preserve-comments \
./node_modules/.bin/_mocha \
--report lcovonly \
-- \
--reporter dot \
--timeout $(TIMEOUT) \
$(MOCHA_OPTS) \
$(TESTS)
test-all: install jshint test cov
autod: install
@./node_modules/.bin/autod -w --prefix "~"
@$(MAKE) install
contributors: install
@./node_modules/.bin/contributors -f plain -o AUTHORS
.PHONY: test

64
test/fixtures/scope-package/README.md vendored Normal file
View File

@@ -0,0 +1,64 @@
scope-package
=======
[![NPM version][npm-image]][npm-url]
[![build status][travis-image]][travis-url]
[![Test coverage][coveralls-image]][coveralls-url]
[![Gittip][gittip-image]][gittip-url]
[![David deps][david-image]][david-url]
[npm-image]: https://img.shields.io/npm/v/scope-package.svg?style=flat
[npm-url]: https://npmjs.org/package/scope-package
[travis-image]: https://img.shields.io/travis/node-modules/scope-package.svg?style=flat
[travis-url]: https://travis-ci.org/node-modules/scope-package
[coveralls-image]: https://img.shields.io/coveralls/node-modules/scope-package.svg?style=flat
[coveralls-url]: https://coveralls.io/r/node-modules/scope-package?branch=master
[gittip-image]: https://img.shields.io/gittip/fengmk2.svg?style=flat
[gittip-url]: https://www.gittip.com/fengmk2/
[david-image]: https://img.shields.io/david/node-modules/scope-package.svg?style=flat
[david-url]: https://david-dm.org/node-modules/scope-package
![logo](https://raw.github.com/node-modules/scope-package/master/logo.png)
scope-package desc
## Install
```bash
$ npm install scope-package
```
## Usage
```js
var scope-package = require('scope-package');
scope-package.foo(function (err) {
});
```
## License
(The MIT License)
Copyright (c) 2014 fengmk2 &lt;fengmk2@gmail.com&gt; and other contributors
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

1
test/fixtures/scope-package/index.js vendored Normal file
View File

@@ -0,0 +1 @@
module.exports = require('./lib/scope-package');

View File

@@ -0,0 +1,17 @@
/**!
* scope-package - lib/scope-package.js
*
* Copyright(c) 2014 fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
"use strict";
/**
* Module dependencies.
*/
var varname = require('modulename');

View File

@@ -0,0 +1,44 @@
{
"name": "@cnpm/scope-package",
"version": "0.0.1",
"description": "scope-package",
"main": "index.js",
"scripts": {
"test": "make test-all"
},
"config": {
"cov": {
"threshold": 100
}
},
"dependencies": {
},
"devDependencies": {
"autod": "*",
"contributors": "*",
"should": "*",
"jshint": "*",
"cov": "*",
"istanbul-harmony": "*",
"mocha": "*"
},
"homepage": "https://github.com/node-modules/scope-package",
"repository": {
"type": "git",
"url": "git://github.com/node-modules/scope-package.git",
"web": "https://github.com/node-modules/scope-package"
},
"bugs": {
"url": "https://github.com/node-modules/scope-package/issues",
"email": "fengmk2@gmail.com"
},
"keywords": [
"scope-package"
],
"engines": {
"node": ">= 0.10.0"
},
"author": "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",
"license": "MIT"
}

41
test/init.js Normal file
View File

@@ -0,0 +1,41 @@
/**!
* cnpmjs.org - test/init.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var crypto = require('crypto');
var utility = require('utility');
var User = require('../proxy/user');
var usernames = [
'cnpmjstest101',
'cnpmjstest10'
];
usernames.forEach(function (name) {
var user = {
name: name,
email: 'fengmk2@gmail.com',
// password: 'cnpmjstest10',
ip: '127.0.0.1'
};
user.salt = crypto.randomBytes(30).toString('hex');
user.password_sha = utility.sha1(user.name + user.salt);
User.add(user, function (err, result) {
if (err) {
throw err;
}
});
});

View File

@@ -34,7 +34,7 @@ describe('middleware/auth.test.js', function () {
describe('auth()', function () {
it('should pass if no authorization', function (done) {
request(app)
.get('/-/user/org.couchdb.user:cnpmjstest1')
.get('/-/user/org.couchdb.user:cnpmjstest10')
.expect(200, done);
});

View File

@@ -0,0 +1,60 @@
/**!
* cnpmjs.org - test/middleware/static.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var request = require('supertest');
var registry = require('../../servers/registry');
var web = require('../../servers/registry');
describe('middleware/static.test.js', function () {
before(function (done) {
registry = registry.listen(0, function () {
web = web.listen(0, done);
});
});
describe('registry', function () {
it('should /favicon.ico rewrite to /favicon.png', function (done) {
request(registry)
.get('/favicon.ico')
// .expect('content-type', 'image/png')
.expect(200, done);
});
it('should 200 /favicon.png', function (done) {
request(registry)
.get('/favicon.png')
.expect('content-type', 'image/png')
.expect(200, done);
});
});
describe('web', function () {
it('should /favicon.ico rewrite to /favicon.png', function (done) {
request(registry)
.get('/favicon.ico')
// .expect('content-type', 'image/png')
.expect(200, done);
});
it('should 200 /favicon.png', function (done) {
request(registry)
.get('/favicon.png')
.expect('content-type', 'image/png')
.expect(200, done);
});
});
});

View File

@@ -16,29 +16,36 @@
var should = require('should');
var request = require('supertest');
var utils = require('../utils');
var app = require('../../servers/web');
var registry = require('../../servers/registry');
describe('middleware/web_not_found.test.js', function () {
before(function (done) {
app.listen(0, done);
});
// make sure mk2testmodule exists
var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
// name: mk2testmodule
var pkg = utils.getPackage('mk2testmodule');
after(function (done) {
app.close(done);
request(registry)
.put('/' + pkg.name)
.set('authorization', utils.adminAuth)
.send(pkg)
.end(done);
});
describe('web_not_found()', function () {
it('should redirect /byte to /package/byte', function (done) {
it('should redirect /mk2testmodule to /package/mk2testmodule', function (done) {
request(app)
.get('/byte')
.expect('Location', '/package/byte')
.get('/mk2testmodule')
.expect('Location', '/package/mk2testmodule')
.expect(302, done);
});
it('should redirect /byte/ to /package/byte', function (done) {
it('should redirect /mk2testmodule/ to /package/mk2testmodule', function (done) {
request(app)
.get('/byte/')
.expect('Location', '/package/byte')
.get('/mk2testmodule/')
.expect('Location', '/package/mk2testmodule')
.expect(302, done);
});
@@ -48,9 +55,9 @@ describe('middleware/web_not_found.test.js', function () {
.expect(404, done);
});
it('should 200 /package/byte', function (done) {
it('should 200 /package/mk2testmodule', function (done) {
request(app)
.get('/package/byte')
.get('/package/mk2testmodule')
.expect(200, done);
});

52
test/proxy/dist.test.js Normal file
View File

@@ -0,0 +1,52 @@
/**!
* cnpmjs.org - test/proxy/dist.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var Dist = require('../../proxy/dist');
describe('proxy/dist.test.js', function () {
describe('savefile() and getfile', function () {
it('should save and get /npm-versions.txt', function* () {
var info = {
name: 'npm-versions.txt',
parent: '/',
date: '15-Sep-2011 23:48',
size: 1676,
url: 'http://cnpmjs.org/dist/npm-versions.txt',
sha1: '104731881047318810473188'
};
yield* Dist.savefile(info);
var got = yield* Dist.getfile('/npm-versions.txt');
should.exist(got);
got.should.eql(info);
});
it('should save and get /v1.0.0/npm-versions.txt', function* () {
var info = {
name: 'npm-versions.txt',
parent: '/v1.0.0/',
date: '15-Sep-2011 23:48',
size: 1676,
url: 'http://cnpmjs.org/dist/v1.0.0/npm-versions.txt',
sha1: '104731881047318810473188'
};
yield* Dist.savefile(info);
var got = yield* Dist.getfile('/v1.0.0/npm-versions.txt');
should.exist(got);
got.should.eql(info);
});
});
});

View File

@@ -38,8 +38,17 @@ describe('proxy/module.test.js', function () {
});
describe('search()', function () {
it('should search modules', function (done) {
Module.search('as', function (err, data) {
before(function (done) {
Module.addKeywords('aaaa', 'mock aaaaaa', ['aa', 'bb', 'cc'], function (err, results) {
should.not.exist(err);
results.should.be.an.Array;
results.should.length(3);
done();
});
});
it.skip('should search modules', function (done) {
Module.search('mock', function (err, data) {
should.not.exist(err);
data.should.have.keys('keywordMatchs', 'searchMatchs');
data.searchMatchs.length.should.above(0);
@@ -74,13 +83,8 @@ describe('proxy/module.test.js', function () {
});
});
var mockName = 'aa' + Date.now();
describe('addKeywords()', function () {
var mockName = 'aa' + Date.now();
after(function (done) {
mysql.query('DELETE FROM module_keyword WHERE name=?', [mockName], done);
});
it('should add diff keywords to module', function (done) {
Module.addKeywords(mockName, mockName, ['aa', 'bb', 'cc'], function (err, results) {
should.not.exist(err);
@@ -91,10 +95,11 @@ describe('proxy/module.test.js', function () {
});
it('should add same keywords to module', function (done) {
Module.addKeywords('aa', 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
Module.addKeywords(mockName, 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
should.not.exist(err);
results.should.be.an.Array;
results.should.length(0);
results.should.length(3);
// results.should.length(0);
done();
});
});
@@ -102,7 +107,7 @@ describe('proxy/module.test.js', function () {
describe('getKeywords()', function () {
it('should get aa module keywords', function (done) {
Module.getKeywords('aa', function (err, keywords) {
Module.getKeywords(mockName, function (err, keywords) {
should.not.exist(err);
keywords.should.eql(['aa', 'bb', 'cc']);
done();
@@ -162,4 +167,10 @@ describe('proxy/module.test.js', function () {
});
});
});
describe('removeTagsByNames()', function () {
it('should work', function* () {
yield* Module.removeTagsByNames('foo', ['latest', '1.0']);
});
});
});

View File

@@ -0,0 +1,69 @@
/**!
* cnpmjs.org - test/proxy/module_maintainer.test.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var ModuleMaintainer = require('../../proxy/module_maintainer');
describe('proxy/module_maintainer.test.js', function () {
describe('update()', function () {
it('should update one maintainer', function* () {
var rs = yield* ModuleMaintainer.update('testfoo', ['fengmk2']);
rs.should.eql({
add: ['fengmk2'],
remove: []
});
// again should be fine
var rs = yield* ModuleMaintainer.update('testfoo', ['fengmk2']);
rs.should.eql({
add: ['fengmk2'],
remove: []
});
// remove the exists
var rs = yield* ModuleMaintainer.update('testfoo', ['fengmk2-1', 'foobar']);
rs.should.eql({
add: ['fengmk2-1', 'foobar'],
remove: ['fengmk2']
});
});
it('should update multi maintainers', function* () {
var rs = yield* ModuleMaintainer.update('testfoo2', ['fengmk2', 'ok', 'foobar']);
rs.should.eql({
add: ['fengmk2', 'ok', 'foobar'],
remove: []
});
// remove exists
var rs = yield* ModuleMaintainer.update('testfoo2', ['fengmk2']);
rs.should.eql({
add: ['fengmk2'],
remove: ['ok', 'foobar']
});
var rs = yield* ModuleMaintainer.update('testfoo3', ['fengmk2', 'ok', 'foobar']);
rs.should.eql({
add: ['fengmk2', 'ok', 'foobar'],
remove: []
});
});
it('should add empty maintainers do nothing', function* () {
var rs = yield* ModuleMaintainer.update('tesfoobar', []);
rs.should.eql({
add: [],
remove: []
});
});
});
});

View File

@@ -22,14 +22,14 @@ var Log = require('../../proxy/module_log');
describe('proxy/sync_module_worker.test.js', function () {
it('should start a sync worker', function (done) {
Log.create({
name: 'cnpmjs.org',
name: 'mk2testmodule',
username: 'fengmk2',
}, function (err, result) {
should.not.exist(err);
result.id.should.above(0);
var worker = new SyncModuleWorker({
logId: result.id,
name: 'cnpmjs.org',
name: 'mk2testmodule',
username: 'fengmk2'
});
@@ -40,25 +40,54 @@ describe('proxy/sync_module_worker.test.js', function () {
it('should start a sync worker with names and noDep', function (done) {
var worker = new SyncModuleWorker({
name: ['cnpmjs.org', 'cutter'],
name: ['mk2testmodule'],
noDep: true,
username: 'fengmk2'
});
worker.start();
worker.on('end', function () {
worker.successes.concat(worker.fails).should.eql(['cnpmjs.org', 'cutter']);
var names = worker.successes.concat(worker.fails);
names.sort();
names.should.eql(['mk2testmodule']);
done();
});
});
it('should start a sync worker with names', function (done) {
var worker = new SyncModuleWorker({
name: ['cnpmjs.org', 'cutter'],
name: ['mk2testmodule'],
username: 'fengmk2'
});
worker.start();
worker.on('end', done);
});
it('should sync unpublished module by name', function* () {
var result = yield* SyncModuleWorker.sync('tnpm', 'fengmk2');
result.ok.should.equal(true);
result.should.have.property('logId');
});
it('should not sync not exists module', function* () {
var result = yield* SyncModuleWorker.sync('tnpm-not-exists', 'fengmk2');
result.ok.should.equal(false);
result.should.not.have.property('logId');
});
it('should sync unpublished info', function (done) {
var worker = new SyncModuleWorker({
name: ['tnpm'],
username: 'fengmk2'
});
worker.start();
worker.on('end', function () {
var names = worker.successes.concat(worker.fails);
names.sort();
names.should.eql(['tnpm']);
done();
});
});
});

View File

@@ -127,11 +127,12 @@ describe('proxy/user.test.js', function () {
describe('update()', function () {
before(initUser);
it('should update ok', function (done) {
user.update(mockUser, function (err, data) {
should.not.exist(err);
should.exist(data);
data.should.have.keys(['rev']);
data.should.have.keys('rev', 'result');
done();
});
});
@@ -140,7 +141,8 @@ describe('proxy/user.test.js', function () {
mockUser.rev = '1-error';
user.update(mockUser, function (err, data) {
should.not.exist(err);
should.not.exist(data);
should.exist(data);
data.result.affectedRows.should.equal(0);
done();
});
});
@@ -176,7 +178,7 @@ describe('proxy/user.test.js', function () {
yield user.saveNpmUser(existUser);
var r = yield mysql.queryOne('select rev, json, npm_user from user where name=?', existUser.name);
should.exist(r);
r.npm_user.should.equal(0);
// r.npm_user.should.equal(0);
r.rev.should.equal(existUser._rev);
JSON.parse(r.json).should.eql(existUser);
});

View File

@@ -14,9 +14,14 @@
* Module dependencies.
*/
var debug = require('debug');
debug.enable('cnpmjs.org*');
var SyncModuleWorker = require('../proxy/sync_module_worker');
var mysql = require('../common/mysql');
var Log = require('../proxy/module_log');
var config = require('../config');
config.sourceNpmRegistry = 'http://registry.npmjs.org';
var names = process.argv[2] || 'byte';
names = names.split(',');

View File

@@ -1,5 +1,5 @@
/*!
* cnpmjs.org - test/sync/sync_all.js
* cnpmjs.org - test/sync/sync_all.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
@@ -20,31 +20,32 @@ var Total = require('../../proxy/total');
var should = require('should');
var Module = require('../../proxy/module');
describe('sync/sync_all.js', function () {
describe('sync/sync_all.test.js', function () {
describe('sync()', function () {
afterEach(mm.restore);
it('should sync first time ok', function *() {
mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
mm.data(Npm, 'getShort', ['mk2testmodule', 'mk2testmodule-not-exists']);
mm.data(Total, 'getTotalInfo', {last_sync_time: 0});
var data = yield sync;
data.successes.should.eql(['cnpmjs.org', 'cutter']);
data.successes.should.eql(['mk2testmodule', 'mk2testmodule-not-exists']);
mm.restore();
var result = yield Total.getTotalInfo();
result.last_sync_module.should.equal('cutter');
should.exist(result);
result.last_sync_module.should.equal('mk2testmodule-not-exists');
});
it('should sync common ok', function *() {
mm.data(Npm, 'getAllSince', {
_updated: Date.now(),
'cnpmjs.org': {},
cutter: {}
'mk2testmodule': {},
// cutter: {}
});
mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
mm.data(Npm, 'getShort', ['mk2testmodule']);
mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
mm.data(Module, 'listAllModuleNames', [{name: 'mk2testmodule'}]);
var data = yield sync;
data.successes.should.eql(['cnpmjs.org', 'cutter']);
data.successes.should.eql(['mk2testmodule']);
mm.restore();
});
});

129
test/sync/sync_dist.test.js Normal file
View File

@@ -0,0 +1,129 @@
/**!
* cnpmjs.org - test/sync/sync_dist.test.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var mm = require('mm');
var urllib = require('co-urllib');
var Dist = require('../../proxy/dist');
var distsync = require('../../sync/sync_dist');
describe('sync/sync_dist.test.js', function () {
afterEach(mm.restore);
describe('listPhantomjsDir()', function () {
it('should list all phantomjs download infos', function* () {
var items = yield* distsync.listPhantomjsDir('/phantomjs');
items.length.should.above(1);
items.forEach(function (item) {
item.should.have.keys('name', 'date', 'size', 'type', 'parent', 'downloadURL');
});
});
});
describe('listdiff()', function () {
it('should got all news', function* () {
mm(urllib, 'request', function* () {
return {
status: 200,
data: '<a href="npm/">npm/</a> 06-May-2014 01:18 -\n<a href="npm-versions.txt">npm-versions.txt</a> 27-Feb-2014 00:01 1676',
headers: {},
};
});
mm(Dist, 'listdir', function* () {
return [];
});
var items = yield* distsync.listdiff('/');
items.should.eql([
{ name: 'npm/',
date: '06-May-2014 01:18',
size: '-',
type: 'dir',
parent: '/' },
{ name: 'npm-versions.txt',
date: '27-Feb-2014 00:01',
size: 1676,
type: 'file',
parent: '/' }
]);
});
it('should got empty when all exists', function* () {
mm(urllib, 'request', function* () {
return {
status: 200,
data: '<a href="npm/">npm/</a> 06-May-2014 01:18 -\n<a href="npm-versions.txt">npm-versions.txt</a> 27-Feb-2014 00:01 1676',
headers: {},
};
});
mm(Dist, 'listdir', function* () {
return [
{
name: 'npm/',
date: '06-May-2014 01:18',
parent: '/'
},
{
name: 'npm-versions.txt',
date: '27-Feb-2014 00:01',
size: 1676,
parent: '/'
},
];
});
var items = yield* distsync.listdiff('/');
items.should.length(0);
});
it('should got date change dir', function* () {
mm(urllib, 'request', function* () {
return {
status: 200,
data: '<a href="npm/">npm/</a> 06-May-2014 01:18 -\n<a href="npm-versions.txt">npm-versions.txt</a> 27-Feb-2014 00:01 1676',
headers: {},
};
});
mm(Dist, 'listdir', function* () {
return [
{
name: 'npm/',
date: '06-May-2014 01:17',
parent: '/'
},
{
name: 'npm-versions.txt',
date: '27-Feb-2014 00:01',
size: 1676,
parent: '/'
},
];
});
var items = yield* distsync.listdiff('/');
items.should.eql([
{ name: 'npm/',
date: '06-May-2014 01:18',
size: '-',
type: 'dir',
parent: '/' }
]);
});
});
});

View File

@@ -1,5 +1,5 @@
/*!
* cnpmjs.org - test/sync/sync_exist.js
* cnpmjs.org - test/sync/sync_exist.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
@@ -13,32 +13,32 @@
/**
* Module dependencies.
*/
var sync = require('../../sync/sync_exist');
var should = require('should');
var mm = require('mm');
var sync = require('../../sync/sync_exist');
var Npm = require('../../proxy/npm');
var Total = require('../../proxy/total');
var should = require('should');
describe('sync/sync_exist.js', function () {
describe('sync/sync_exist.test.js', function () {
describe('sync()', function () {
afterEach(mm.restore);
it('should sync first time ok', function *() {
mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
mm.data(Npm, 'getShort', ['mk2testmodule']);
mm.data(Total, 'getTotalInfo', {last_exist_sync_time: 0});
var data = yield sync();
data.successes.should.eql(['cnpmjs.org', 'cutter']);
data.successes.should.eql(['mk2testmodule']);
});
it('should sync common ok', function *() {
mm.data(Npm, 'getAllSince', {
_updated: Date.now(),
'cnpmjs.org': {},
cutter: {}
'mk2testmodule': {},
});
mm.data(Total, 'getTotalInfo', {last_exist_sync_time: Date.now()});
var data = yield sync();
data.successes.should.eql(['cnpmjs.org', 'cutter']);
data.successes.should.eql(['mk2testmodule']);
});
});
});

25
test/sync_dist.js Normal file
View File

@@ -0,0 +1,25 @@
/**!
* cnpmjs.org - test/sync_dist.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var debug = require('debug');
debug.enable('cnpmjs.org*');
var co = require('co');
var syncdist = require('../sync/sync_dist');
co(function* () {
// yield* syncdist('/v0.10.28/');
yield* syncdist.syncPhantomjsDir();
})();

48
test/utils.js Normal file
View File

@@ -0,0 +1,48 @@
/**!
* cnpmjs.org - test/utils.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var path = require('path');
var fs = require('fs');
var fixtures = path.join(__dirname, 'fixtures');
var admin = exports.admin = 'cnpmjstest10';
exports.adminAuth = 'Basic ' + new Buffer(admin + ':' + admin).toString('base64');
var otherUser = exports.otherUser = 'cnpmjstest101';
exports.otherUserAuth = 'Basic ' + new Buffer(otherUser + ':' + otherUser).toString('base64');
var _pkg = fs.readFileSync(path.join(fixtures, 'package_and_tgz.json'));
exports.getPackage = function (name, version, user) {
// name: mk2testmodule
name = name || 'mk2testmodule';
version = version || '0.0.1';
user = user || admin;
var pkg = JSON.parse(_pkg);
var versions = pkg.versions;
pkg.versions = {};
pkg.versions[version] = versions[Object.keys(versions)[0]];
pkg.maintainers[0].name = user;
pkg.versions[version].maintainers[0].name = user;
pkg.versions[version].name = name;
pkg.versions[version].version = version;
pkg.versions[version]._id = name + '@' + version;
pkg.name = name;
pkg['dist-tags'] = {latest: version};
return pkg;
};

View File

@@ -0,0 +1,9 @@
var zlib = require('zlib');
var stream = zlib.createGzip();
stream.write('foo');
stream.close();
// Assertion failed: (!ctx->pending_close_ && "close is pending"), function Write, file ../src/node_zlib.cc, line 150.
stream.write('again');

5
view/web/404.html Normal file
View File

@@ -0,0 +1,5 @@
<div class="alert alert-warning">
Can not found package match <%= name %>. You can
<a href="/sync/<%= name %>" target="_blank">SYNC</a> from official npm registry or
<a href="https://npmjs.org/search?q=<%= name %>" target="_blank">SEARCH</a> in official npm website.
</div>

9
view/web/dist.html Normal file
View File

@@ -0,0 +1,9 @@
<h1>Mirror index of <a target="_blank" href="<%= disturl %><%= dirname %>"><%= disturl %><%= dirname %></a></h1>
<hr>
<pre><a href="../">../</a><% for (var i = 0; i < items.length; i++) {
var item = items[i];
var sizestr = '' + (item.size || '-');
%>
<a href="<%= item.name %>"><%= item.name %></a><%- padding(50, item.name.length, " ") %><%= item.date %><%- padding(20, sizestr.length, " ") %><%= sizestr %><% } %>
</pre>
<hr>

View File

@@ -4,12 +4,12 @@
<meta charset="utf-8">
<title><%- locals.title %></title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="shortcut icon" href="/public/favicon.png">
<link rel="shortcut icon" href="/favicon.png">
<!-- Bootstrap -->
<link href="http://cdn.staticfile.org/twitter-bootstrap/3.0.0-rc2/css/bootstrap.min.css" rel="stylesheet" media="screen">
<link href="http://cdn.staticfile.org/prettify/r298/prettify.min.css" rel="stylesheet" media="screen">
<link href="//dn-staticfile.qbox.me/twitter-bootstrap/3.0.0-rc2/css/bootstrap.min.css" rel="stylesheet" media="screen">
<link href="//dn-staticfile.qbox.me/prettify/r298/prettify.min.css" rel="stylesheet" media="screen">
<!-- JavaScript plugins (requires jQuery) -->
<script src="http://cdn.staticfile.org/jquery/2.0.3/jquery.min.js"></script>
<script src="//dn-staticfile.qbox.me/jquery/2.0.3/jquery.min.js"></script>
<link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="CNPM" />
<style>
a{color:#09f;}
@@ -72,17 +72,17 @@
{{footer}}
</p>
<a href="https://github.com/cnpm/cnpmjs.org" id="fork" target="_blank">
<img alt="Fork me on GitHub" src="http://s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png">
<img alt="Fork me on GitHub" src="//s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png">
</a>
</div>
</div>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="http://cdn.staticfile.org/twitter-bootstrap/3.0.0-rc2/js/bootstrap.min.js"></script>
<script src="//dn-staticfile.qbox.me/twitter-bootstrap/3.0.0-rc2/js/bootstrap.min.js"></script>
<!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) -->
<script src="http://cdn.staticfile.org/respond.js/1.2.0/respond.min.js"></script>
<script src="//dn-staticfile.qbox.me/respond.js/1.2.0/respond.min.js"></script>
<script src="http://cdn.staticfile.org/prettify/r298/prettify.min.js"></script>
<script src="//dn-staticfile.qbox.me/prettify/r298/prettify.min.js"></script>
<!-- Specific to this page -->
<script>

View File

@@ -85,6 +85,18 @@
<% } %>
</td>
</tr>
<% if (package.engines) {%>
<tr>
<th>Engines</th>
<td>
<ul>
<% for (var k in package.engines) { %>
<li><%= k %>: <%= package.engines[k] %></li>
<% } %>
</ul>
</td>
</tr>
<% } %>
<% if (package.license) { %>
<tr>
<th>License</th>
@@ -240,6 +252,7 @@
<a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
<%= package.dist.tarball %>
</a>
(<%- package.dist.size %>)
</td>
</tr>
<% } %>

View File

@@ -20,7 +20,7 @@
<a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
<a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
</div>
<% } else if (packages[0].name !== keyword) { %>
<% } else if (!match) { %>
<div class="alert alert-info">
Can not found package <%= keyword %>. You can
<a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
@@ -32,10 +32,19 @@
Packages match "<span style="color: #09f;"><%= keyword %></span>"
</h1>
<hr />
<% if (match) { %>
<div class="package match">
<a href="/package/<%= match.name %>" class="package-name"><%= match.name %></a>
<span class="package-description"><%= match.description %></span>
</div>
<% } %>
<% for (var i = 0; i < packages.length; i++) {
var item = packages[i];
if (item.name === keyword) {
continue;
}
%>
<div class="package <%= item.name === keyword ? 'match' : '' %>">
<div class="package">
<a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
<span class="package-description"><%= item.description %></span>
</div>

View File

@@ -21,12 +21,14 @@ var config = require('./config');
var registry = require('./servers/registry');
var web = require('./servers/web');
registry.listen(config.registryPort);
web.listen(config.webPort);
registry.listen(config.registryPort, config.bindingHost);
web.listen(config.webPort, config.bindingHost);
console.log('[%s] [worker:%d] Server started, registry server listen at %d, web listen at %d, cluster: %s',
console.log('[%s] [worker:%d] Server started, registry server listen at %s:%d, web listen at %s:%d, cluster: %s',
new Date(), process.pid,
config.registryPort, config.webPort, config.enableCluster);
config.bindingHost, config.registryPort,
config.bindingHost, config.webPort,
config.enableCluster);
graceful({
server: [registry, web],