Compare commits

...

144 Commits
0.3.4 ... 0.4.3

Author SHA1 Message Date
dead_horse
b1b6172892 Release 0.4.3 2014-04-18 16:15:50 +08:00
fengmk2
e5a77a4368 Merge pull request #334 from cnpm/fix-permission
add permission check to /:name/:tag
2014-04-18 14:34:27 +08:00
dead_horse
b74cccd342 add permission check to /:name/:tag 2014-04-18 14:29:42 +08:00
fengmk2
6aae538f49 Merge pull request #333 from cnpm/issue332-tag
add put /:name/:tag, close #332
2014-04-18 14:12:56 +08:00
dead_horse
74c8b25374 fix space 2014-04-18 14:07:49 +08:00
dead_horse
0cec2edea6 add put /:name/:tag, close #332 2014-04-18 12:18:07 +08:00
fengmk2
b59a0a99cf Release 0.4.2 2014-04-17 17:33:17 +08:00
dead_horse
c7e82809e3 Merge pull request #330 from cnpm/package-size-show
fix fav ico and show pkg size on pkg info page. fix #318
2014-04-17 17:31:06 +08:00
fengmk2
206ee505a8 sync interval config 2014-04-17 17:28:27 +08:00
fengmk2
d39838f930 fix fav ico and show pkg size on pkg info page. fix #318 2014-04-17 14:59:38 +08:00
dead_horse
512e21aaf5 Merge pull request #329 from cnpm/fix-stack-over
sync work sync one done must wait for a defer.setImmediate. fix #328
2014-04-17 14:12:22 +08:00
fengmk2
f7904ee699 sync work sync one done must wait for a defer.setImmediate. fix #328 2014-04-17 09:11:46 +08:00
dead_horse
7007be6ef4 bump dep versions 2014-04-16 15:17:14 +08:00
dead_horse
70aa3299e5 Merge pull request #326 from cnpm/noattachment
if download tarball 404, throw err better than ignore it. fixed #325
2014-04-16 12:00:29 +08:00
fengmk2
a04f5d68eb if download tarball 404, throw err better than ignore it. fixed #325 2014-04-16 11:44:28 +08:00
fengmk2
7cfe3b58ce refator sync 2014-04-16 11:19:32 +08:00
fengmk2
afdf0bc653 Merge pull request #323 from cnpm/hot-fix-status
hotfix, close #319
2014-04-16 09:39:15 +08:00
fengmk2
42055ac91e Merge pull request #324 from cnpm/hotfix-dist
hotfix, close #321
2014-04-16 09:31:48 +08:00
dead_horse
b93e0dab41 hotfix, close #321 2014-04-16 09:28:22 +08:00
dead_horse
01f2187830 hotfix, close #319 2014-04-16 09:23:59 +08:00
fengmk2
c2f49fcdd9 support custom web home page 2014-04-14 15:14:28 +08:00
dead_horse
3925ef6044 Merge pull request #316 from cnpm/first-full-sync
npm get short only can read from cnpm now
2014-04-14 14:04:59 +08:00
fengmk2
85494d2ba0 npm get short only can read from cnpm now 2014-04-14 13:57:04 +08:00
dead_horse
660ca394d6 Merge pull request #315 from cnpm/local-bindding
Local bindding and redis fix
2014-04-14 10:47:39 +08:00
fengmk2
ad7eeac00c if using reverted proxy like nginx, only binding on local host 2014-04-14 10:43:49 +08:00
fengmk2
f63b72891b fix redis detect logic 2014-04-14 10:27:46 +08:00
fengmk2
28d31b093d Release 0.4.1 2014-04-10 14:46:01 +08:00
fengmk2
466e14e92a Merge pull request #313 from cnpm/hotfix-sync-status
fix sync status code error
2014-04-10 07:46:57 +08:00
dead_horse
3646c2ce50 fix sync status code error 2014-04-10 02:39:02 +08:00
fengmk2
fa44097d8f Release 0.4.0 2014-04-09 21:05:37 +08:00
dead_horse
0c56294c47 Merge pull request #312 from cnpm/local-mysql-dev-env
Local mysql dev env
2014-04-09 20:53:12 +08:00
fengmk2
41552fd27c fix test cases to run on local machine 2014-04-09 19:37:37 +08:00
fengmk2
cb4d4f51dd add contribute guidelines 2014-04-09 09:58:23 +08:00
fengmk2
1b266d527c use local mysql for dev env. fix #308 2014-04-09 09:31:34 +08:00
fengmk2
9d660be259 Merge pull request #306 from cnpm/use-copy-to
use copy to
2014-04-01 17:37:36 +08:00
dead_horse
b041cc7361 use copy to 2014-04-01 16:19:49 +08:00
dead_horse
e420f6985e Merge pull request #305 from cnpm/fix-test
fix test case
2014-03-31 17:51:15 +08:00
fengmk2
60dc5cf5fa fix test case 2014-03-31 17:47:10 +08:00
fengmk2
54916f49e5 Merge pull request #303 from cnpm/compress-conditional
use koa-compress and koa-conditional-get
2014-03-31 14:52:39 +08:00
dead_horse
a3cafaa297 use koa-compress and koa-conditional-get 2014-03-31 14:21:13 +08:00
fengmk2
aa13a100d4 Merge pull request #302 from cnpm/typeerror-hotfix
maintainers is string, fix #301
2014-03-30 09:01:23 +08:00
fengmk2
3b29310826 maintainers is string, fix #301 2014-03-30 09:01:03 +08:00
fengmk2
6d76a590e5 Release 0.3.13 2014-03-27 16:28:01 +08:00
fengmk2
3a98b63736 Merge pull request #299 from cnpm/fix-user-update
fix npm adduser update 409 bug
2014-03-27 16:24:34 +08:00
fengmk2
f5a2090fda Merge pull request #298 from cnpm/fix-ml-cov
fix multiline coverage
2014-03-27 16:18:23 +08:00
fengmk2
989a69143f fix npm adduser update 409 bug 2014-03-27 16:18:01 +08:00
dead_horse
77e6db4f0e fix multiline coverage 2014-03-27 16:03:00 +08:00
fengmk2
a3f7affe7b Release 0.3.13 2014-03-27 15:47:19 +08:00
dead_horse
2cd0453c6b Merge pull request #297 from cnpm/pkg-engine
show package engines. fixed #280
2014-03-27 15:23:32 +08:00
fengmk2
84bc126d57 show package engines. fixed #280 2014-03-27 15:13:18 +08:00
dead_horse
760421661c Merge pull request #296 from cnpm/sync-with-local
dont sync local package field. fix #295
2014-03-27 15:04:25 +08:00
fengmk2
425e430a3a dont sync local package field. fix #295 2014-03-27 14:55:52 +08:00
fengmk2
ad3602bf15 Release 0.3.12 2014-03-26 15:28:05 +08:00
fengmk2
2c5852b344 Merge pull request #294 from cnpm/hotfix-sync
fix result.successes not exist error
2014-03-26 15:22:59 +08:00
dead_horse
115a349fc5 fix result.successes not exist error 2014-03-26 15:21:37 +08:00
fengmk2
3ceea3ccf4 Merge pull request #293 from cnpm/search-list
fix search list
2014-03-24 14:20:51 +08:00
dead_horse
ab6a9b2bea fix search list 2014-03-24 14:17:22 +08:00
fengmk2
9cc84abbb5 Merge pull request #292 from cnpm/issue291-all
only return package name in /-/all and /-/all/since, fixed #291
2014-03-24 11:27:31 +08:00
dead_horse
c1c07cf2e4 add simple request for listall 2014-03-24 11:11:54 +08:00
dead_horse
ce5c97956b only return package name in /-/all and /-/all/since, fixed #291 2014-03-24 10:31:28 +08:00
dead_horse
7cb7a517da refine docs foloder 2014-03-23 02:28:03 +08:00
dead_horse
c7df6f0f1a fix typo 2014-03-23 02:26:38 +08:00
fengmk2
ee59176b67 Merge pull request #290 from cnpm/modified-as-etag
use module gmt_modified as etag. fix #288
2014-03-22 02:09:42 +08:00
fengmk2
f160d741cb use module gmt_modified as etag. fix #288 2014-03-22 01:55:13 +08:00
dead_horse
94974f81ae fix typo, remove unused config in package.json 2014-03-21 17:26:54 +08:00
fengmk2
41c0fea3dd Merge pull request #289 from cnpm/readme
Readme
2014-03-21 16:58:27 +08:00
dead_horse
dfae519e96 fix typo 2014-03-21 16:56:15 +08:00
dead_horse
4d7d4f06b1 web page only list cnpm registry related info 2014-03-21 16:25:47 +08:00
dead_horse
5a890ff27e update readme 2014-03-21 16:15:38 +08:00
fengmk2
ac124d0e2d Merge pull request #286 from cnpm/qnfs-generator
use generator in qnfs
2014-03-20 21:17:03 +08:00
dead_horse
7e267156eb use generator in qnfs 2014-03-20 20:35:58 +08:00
fengmk2
258c34fea8 Release 0.3.11 2014-03-20 10:41:37 +08:00
fengmk2
cd9d403237 Merge pull request #284 from cnpm/issue283-maintainer
use common.isMaintainer, fixed #283
2014-03-20 10:35:55 +08:00
dead_horse
6d43612044 use common.isMaintainer, fixed #283 2014-03-20 09:49:47 +08:00
fengmk2
2e99a779c7 Merge pull request #282 from cnpm/update-thunkify
update thunkify-wrap, breaking change in thunkify-wrap
2014-03-18 10:49:39 +08:00
dead_horse
43259ed126 update dependencies 2014-03-18 10:44:21 +08:00
fengmk2
51e32e3ece Merge pull request #281 from cnpm/issue279-co-mocha
use co-mocha for test, fixed #279
2014-03-18 09:35:39 +08:00
dead_horse
37d9d62d3f use co-mocha for test, fixed #279 2014-03-17 22:27:43 +08:00
dead_horse
2e0a1ca3e8 update thunkify-wrap, breaking change in thunkify-wrap 2014-03-17 17:36:37 +08:00
fengmk2
5abaee4f2a Merge pull request #278 from cnpm/use-multiline
Use multiline
2014-03-17 14:06:59 +08:00
dead_horse
5e2ca1d5e8 refactor SQLs by using multiline 2014-03-17 13:27:09 +08:00
dead_horse
c0a6bb4869 fix typo 2014-03-16 22:28:56 +08:00
dead_horse
2aba0836cf use multiline to refactor sqls 2014-03-16 22:20:11 +08:00
fengmk2
e1636e2c7d ignore contributors 2014-03-16 16:43:58 +08:00
fengmk2
15245dc68b Release 0.3.10 2014-03-16 16:33:02 +08:00
dead_horse
8dedabaa2d Merge pull request #277 from cnpm/only-authsession-send-cookie
Only /_session request send the authSession. fixed #223
2014-03-16 16:24:35 +08:00
fengmk2
f4b82bbf0b Only /_session request send the authSession. fixed #223
* no auth requests url dont send the authSession
* baseauth dont create session cookie too
2014-03-16 16:14:12 +08:00
dead_horse
78d81c02cd Merge pull request #276 from cnpm/sync-npm-user
Sync npm user
2014-03-16 15:28:31 +08:00
fengmk2
1c743f3759 sync npm user info when maintainers and contributors not exists. fixed #82 2014-03-16 14:58:44 +08:00
fengmk2
041f80ab14 save npm user to mysql 2014-03-16 12:44:38 +08:00
fengmk2
e5646c6553 password salt always be randoms 2014-03-16 11:51:06 +08:00
fengmk2
6134983bcc Merge pull request #275 from cnpm/issue274-sync-by-install
remove session access in /name and /name/version, fixed #274
2014-03-16 11:36:27 +08:00
dead_horse
7dab569d43 remove session access in /name and /name/version, fixed #274 2014-03-15 23:20:34 +08:00
fengmk2
5aa84b03c6 Merge pull request #270 from cnpm/update-session
Update session
2014-03-15 22:59:23 +08:00
dead_horse
77fc133ecb fix update maintainer session error 2014-03-15 22:56:03 +08:00
dead_horse
f98416e217 update koa-middlewares 2014-03-15 22:52:50 +08:00
dead_horse
5f49b97aa4 fix test, fix sync_by_install 2014-03-15 22:52:50 +08:00
dead_horse
8bdfd593c9 use defer session 2014-03-15 22:52:48 +08:00
dead_horse
c214da39dc Merge pull request #272 from cnpm/author-add
Support npm owner|author add [name] [pkg]. fixed #271
2014-03-15 22:41:34 +08:00
fengmk2
f2be453ef6 Support npm owner|author add [name] [pkg]. fixed #271
* npm 1.4.4+ adduser new flow support
* npm author add, rm, ls support
2014-03-15 17:15:42 +08:00
fengmk2
e2b7003749 Release 0.3.9 2014-03-14 15:05:45 +08:00
fengmk2
944ac6a712 custom user-agent 2014-03-14 15:04:29 +08:00
dead_horse
c502c00063 Merge pull request #269 from cnpm/co-urllib
use co-urllib instead of thunkify urllib; fix mock http.request test cases
2014-03-14 14:42:12 +08:00
fengmk2
d536ad954b use co-urllib instead of thunkify urllib; fix mock http.request test cases 2014-03-14 14:33:20 +08:00
fengmk2
100706f311 request limit custom message 2014-03-13 10:47:06 +08:00
fengmk2
8d8fd31a0e Merge pull request #268 from cnpm/issue267-limit
add koa-limit, fixed #267
2014-03-13 09:55:30 +08:00
dead_horse
7aa2ea3086 add config.redis check 2014-03-13 09:31:30 +08:00
dead_horse
5bf40164a1 add koa-limit, fixed #267 2014-03-13 01:34:36 +08:00
fengmk2
e497fb354d Release 0.3.8 2014-03-11 15:13:11 +08:00
fengmk2
e647d4a1fe Merge pull request #265 from cnpm/update-middlewares
update middlewares, fixed 264
2014-03-11 15:08:33 +08:00
dead_horse
33465ecf86 update middlewares, fixed 264 2014-03-11 15:06:56 +08:00
fengmk2
3eb3eb6be9 Release 0.3.7 2014-03-11 12:02:39 +08:00
fengmk2
6ab1556ec0 show worker die date time 2014-03-10 13:50:38 +08:00
fengmk2
9fd21a5c25 Merge pull request #261 from cnpm/update-koa
update to koa@0.5.1
2014-03-08 10:35:20 +08:00
dead_horse
53b8398e93 update to koa@0.5.1 2014-03-08 10:30:27 +08:00
fengmk2
c43b117e67 hotfix for star user 2014-03-08 00:23:11 +08:00
fengmk2
7db269e5f4 Merge pull request #253 from cnpm/issue237-sync-worker
Issue237 sync worker
2014-03-07 23:48:09 +08:00
dead_horse
92ed8984d3 fix yield gather, sync missing deps even no missing versions 2014-03-07 23:45:13 +08:00
dead_horse
71d351da29 fix return versions 2014-03-07 22:36:13 +08:00
dead_horse
c8e1d8cc4a fix makefile, remove eventproxy 2014-03-07 22:36:13 +08:00
dead_horse
fd79879d0c refactor sync_module_worker 2014-03-07 22:36:13 +08:00
dead_horse
354b82f6ea refactor sync_module_worker 2014-03-07 22:36:13 +08:00
dead_horse
de0487cccc add make test-dev, fixed #259 2014-03-07 22:35:15 +08:00
dead_horse
081ccd66df change npm.js to generator 2014-03-07 22:35:15 +08:00
dead_horse
cebf501182 update urllib, proxy/npm.js use generator
but SyncModuleWorker still use callback type npm.js,
so just use co wrap to thunk.
after refactor SyncModuleWorker, remove this co wrap.
2014-03-07 22:35:14 +08:00
dead_horse
8fe593a182 sync_all and sync_exist to generator 2014-03-07 22:34:43 +08:00
dead_horse
8770458a46 change function to generator 2014-03-07 22:34:43 +08:00
dead_horse
559dc03cd4 need node >= v0.11.9 2014-03-06 16:16:36 +08:00
fengmk2
70d12f235a Release 0.3.6 2014-03-06 15:05:07 +08:00
fengmk2
aa7077e4b0 install missing package should sync it from source npm. fixed #252 2014-03-06 15:04:07 +08:00
dead_horse
f5c19668f5 Merge pull request #254 from cnpm/jshint
Add jshint check: $ make jshint
2014-03-06 14:09:07 +08:00
fengmk2
2b2a1904c6 npm publish dont contains .jshint* 2014-03-06 14:02:33 +08:00
fengmk2
a234c7e289 npm test run jshint 2014-03-06 14:00:43 +08:00
fengmk2
63562c30ec Add jshint check: $ make jshint
* .jshint make esnext true
* .jshintignore ignore no need check dirs
2014-03-06 13:55:13 +08:00
dead_horse
6968331b38 Merge pull request #251 from cnpm/yield-next
use `yield* next` instead of `yield next`
2014-03-05 16:55:32 +08:00
fengmk2
fc1a17e4d1 use yield* next instead of yield next
* According to koajs/compose#2, show performances improve a lot.
2014-03-05 16:49:58 +08:00
fengmk2
3872b7dfdf replace dist.u.qiniudn.com with cnpmjs.org/dist 2014-03-05 13:20:45 +08:00
fengmk2
e340ce081a Release 0.3.5 2014-03-05 13:18:39 +08:00
dead_horse
170a689168 Merge pull request #250 from cnpm/dist
redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249
2014-03-05 11:08:05 +08:00
fengmk2
7794d1099d redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249 2014-03-05 11:00:41 +08:00
dead_horse
bd8686c448 Merge pull request #247 from cnpm/redirect
redirect /name to /package/name when /name is 404. fixed #245
2014-03-04 23:54:48 +08:00
fengmk2
d8a76d03e0 redirect /name to /package/name when /name is 404. fixed #245 2014-03-04 23:22:32 +08:00
dead_horse
4460810799 Merge pull request #244 from cnpm/missing-propertys
Add missing properies and sync missing star users. fixed #235
2014-03-04 21:35:57 +08:00
fengmk2
4124525eb8 Add missing properies and sync missing star users. fixed #235
* add readmeFilename and missing created, modified time.
* Sync star users on pkg sync flow
* Show star users on registry.show
* Show star users on web package show
* Need to create new table module_star
2014-03-04 21:20:43 +08:00
81 changed files with 3103 additions and 1199 deletions

4
.gitignore vendored
View File

@@ -7,6 +7,7 @@ coverage.html
*.pid
*.gz
dump.rdb
.DS_Store
pids
logs
@@ -15,11 +16,14 @@ results
node_modules
npm-debug.log
public/dist/
.dist
config/config.js
backup/*.json
backup/*.gz
docs/web/history.md
docs/web/_readme.md
view/web/_layout.html
bin/mysql.js
bin/test.sql
coverage/
config/web_readme.md

4
.jshintignore Normal file
View File

@@ -0,0 +1,4 @@
node_modules/
coverage/
.tmp/
.git/

95
.jshintrc Normal file
View File

@@ -0,0 +1,95 @@
{
// JSHint Default Configuration File (as on JSHint website)
// See http://jshint.com/docs/ for more details
"maxerr" : 50, // {int} Maximum error before stopping
// Enforcing
"bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
"camelcase" : false, // true: Identifiers must be in camelCase
"curly" : true, // true: Require {} for every new block or scope
"eqeqeq" : true, // true: Require triple equals (===) for comparison
"forin" : false, // true: Require filtering for..in loops with obj.hasOwnProperty()
"immed" : false, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
"indent" : false, // {int} Number of spaces to use for indentation
"latedef" : false, // true: Require variables/functions to be defined before being used
"newcap" : false, // true: Require capitalization of all constructor functions e.g. `new F()`
"noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee`
"noempty" : true, // true: Prohibit use of empty blocks
"nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment)
"plusplus" : false, // true: Prohibit use of `++` & `--`
"quotmark" : false, // Quotation mark consistency:
// false : do nothing (default)
// true : ensure whatever is used is consistent
// "single" : require single quotes
// "double" : require double quotes
"undef" : true, // true: Require all non-global variables to be declared (prevents global leaks)
"unused" : false, // true: Require all defined variables be used
"strict" : true, // true: Requires all functions run in ES5 Strict Mode
"trailing" : false, // true: Prohibit trailing whitespaces
"maxparams" : false, // {int} Max number of formal params allowed per function
"maxdepth" : false, // {int} Max depth of nested blocks (within functions)
"maxstatements" : false, // {int} Max number statements per function
"maxcomplexity" : false, // {int} Max cyclomatic complexity per function
"maxlen" : false, // {int} Max number of characters per line
// Relaxing
"asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons)
"boss" : true, // true: Tolerate assignments where comparisons would be expected
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
"eqnull" : false, // true: Tolerate use of `== null`
"es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
"esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`)
"moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// (ex: `for each`, multiple try/catch, function expression…)
"evil" : false, // true: Tolerate use of `eval` and `new Function()`
"expr" : true, // true: Tolerate `ExpressionStatement` as Programs
"funcscope" : false, // true: Tolerate defining variables inside control statements"
"globalstrict" : false, // true: Allow global "use strict" (also enables 'strict')
"iterator" : false, // true: Tolerate using the `__iterator__` property
"lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block
"laxbreak" : true, // true: Tolerate possibly unsafe line breakings
"laxcomma" : false, // true: Tolerate comma-first style coding
"loopfunc" : false, // true: Tolerate functions being defined in loops
"multistr" : true, // true: Tolerate multi-line strings
"proto" : false, // true: Tolerate using the `__proto__` property
"scripturl" : false, // true: Tolerate script-targeted URLs
"smarttabs" : false, // true: Tolerate mixed tabs/spaces when used for alignment
"shadow" : true, // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
"sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation
"supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;`
"validthis" : false, // true: Tolerate using this in a non-constructor function
// Environments
"browser" : true, // Web Browser (window, document, etc)
"couch" : false, // CouchDB
"devel" : true, // Development/debugging (alert, confirm, etc)
"dojo" : false, // Dojo Toolkit
"jquery" : false, // jQuery
"mootools" : false, // MooTools
"node" : true, // Node.js
"nonstandard" : false, // Widely adopted globals (escape, unescape, etc)
"prototypejs" : false, // Prototype and Scriptaculous
"rhino" : false, // Rhino
"worker" : false, // Web Workers
"wsh" : false, // Windows Scripting Host
"yui" : false, // Yahoo User Interface
"noyield" : true, // allow generators without a yield
// Legacy
"nomen" : false, // true: Prohibit dangling `_` in variables
"onevar" : false, // true: Allow only one `var` statement per function
"passfail" : false, // true: Stop on first error
"white" : false, // true: Check against strict whitespace and indentation rules
// Custom Globals
"globals" : { // additional predefined global variables
// mocha
"describe": true,
"it": true,
"before": true,
"afterEach": true,
"beforeEach": true,
"after": true
}
}

View File

@@ -8,6 +8,13 @@ logo.png
public/dist/
backup/*.json
backup/*.gz
docs/web/history.md
docs/web/_readme.md
view/web/_layout.html
bin/mysql.js
bin/test.sql
coverage/
.jshintrc
.jshintignore
.DS_Store
config/web_readme.md

39
CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,39 @@
# How to contribute
Third-party patches are essential for keeping `cnpmjs.org` great.
We want to keep it as easy as possible to contribute changes that
get things working in your environment. There are a few guidelines that we
need contributors to follow so that we can have a chance of keeping on
top of things.
## Getting Started
* Make sure you have a [GitHub account](https://github.com/signup/free)
* Fork the repository on GitHub
## Making Changes
* Create a topic branch from where you want to base your work.
* This is usually the master branch.
* Only target release branches if you are certain your fix must be on that
branch.
* To quickly create a topic branch based on master.
Please avoid working directly on the `master` branch.
* Make commits of logical units and including unit tests.
* Check for unnecessary whitespace with `git diff --check` before committing.
* Make sure your commit messages are in the proper format.
* Make sure you have added the necessary tests for your changes.
* Run _all_ the tests to assure nothing else was accidentally broken.
* Follow [node style guide](https://github.com/felixge/node-style-guide)
## Submitting Changes
* Push your changes to a topic branch in your fork of the repository.
* Submit a pull request.
* Make sure travis-ci test pass.
# Additional Resources
* [General GitHub documentation](http://help.github.com/)
* [GitHub pull request documentation](http://help.github.com/send-pull-requests/)
* [cnpmjs.org](http://cnpmjs.org)

View File

@@ -1,4 +1,138 @@
0.4.3 / 2014-04-18
==================
* Merge pull request #334 from cnpm/fix-permission
* add permission check to /:name/:tag
* Merge pull request #333 from cnpm/issue332-tag
* fix space
* add put /:name/:tag, close #332
0.4.2 / 2014-04-17
==================
* sync interval config
* fix fav ico and show pkg size on pkg info page. fix #318
* sync work sync one done must wait for a defer.setImmediate. fix #328
* bump dep versions
* if download tarball 404, throw err better than ignore it. fixed #325
* refator sync
* hotfix, close #321
* hotfix, close #319
* support custom web home page
* npm get short only can read from cnpm now
* if using reverted proxy like nginx, only binding on local host
* fix redis detect logic
0.4.1 / 2014-04-10
==================
* fix sync status code error
0.4.0 / 2014-04-09
==================
* fix test cases to run on local machine
* add contribute guidelines
* use local mysql for dev env. fix #308
* use copy to
* use koa-compress and koa-conditional-get
* maintainers is string, fix #301
0.3.13 / 2014-03-27
==================
* fix npm adduser update 409 bug
* fix multiline coverage
* show package engines. fixed #280
* dont sync local package field. fix #295
0.3.12 / 2014-03-26
==================
* fix result.successes not exist error
* fix search list
* add simple request for listall
* only return package name in /-/all and /-/all/since, fixed #291
* refine docs foloder
* use module gmt_modified as etag. fix #288
* fix typo, remove unused config in package.json
* web page only list cnpm registry related info
* use generator in qnfs
0.3.11 / 2014-03-20
==================
* use common.isMaintainer, fixed #283
* update dependencies
* use co-mocha for test, fixed #279
* update thunkify-wrap, breaking change in thunkify-wrap
* refactor SQLs by using multiline
* use multiline to refactor sqls
* ignore contributors
0.3.10 / 2014-03-16
==================
* Only /_session request send the authSession. fixed #223
* sync npm user info when maintainers and contributors not exists. fixed #82
* save npm user to mysql
* password salt always be randoms
* remove session access in /name and /name/version, fixed #274
* fix update maintainer session error
* update koa-middlewares
* fix test, fix sync_by_install
* use defer session
* Support npm owner|author add [name] [pkg]. fixed #271
0.3.9 / 2014-03-14
==================
* custom user-agent
* use co-urllib instead of thunkify urllib; fix mock http.request test cases
* request limit custom message
* add config.redis check
* add koa-limit, fixed #267
0.3.8 / 2014-03-11
==================
* update middlewares, fixed missing charset bug #264
0.3.7 / 2014-03-11
==================
* show worker die date time
* update to koa@0.5.1
* hotfix for star user
* fix yield gather, sync missing deps even no missing versions
* fix return versions
* fix makefile, remove eventproxy
* refactor sync_module_worker
* add make test-dev, fixed #259
* change npm.js to generator
* update urllib, proxy/npm.js use generator
* sync_all and sync_exist to generator
* change function to generator
* need node >= v0.11.9
0.3.6 / 2014-03-06
==================
* install missing package should sync it from source npm. fixed #252
* npm publish dont contains .jshint*
* npm test run jshint
* Add jshint check: $ make jshint
* use `yield* next` instead of `yield next`
* replace dist.u.qiniudn.com with cnpmjs.org/dist
0.3.5 / 2014-03-05
==================
* redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249
* redirect /name to /package/name when /name is 404. fixed #245
* Add missing properies and sync missing star users. fixed #235
0.3.4 / 2014-03-04
==================

View File

@@ -2,36 +2,51 @@ TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
REPORTER = tap
TIMEOUT = 30000
MOCHA_OPTS =
REGISTRY = --registry=http://r.cnpmjs.org
install:
@npm install --registry=http://registry.cnpmjs.org \
--cache=${HOME}/.npm/.cache/cnpm --disturl=http://dist.u.qiniudn.com
@npm install $(REGISTRY) \
--disturl=http://dist.cnpmjs.org
test:
jshint: install
@-./node_modules/.bin/jshint ./
pretest:
@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
@mysql -uroot 'cnpmjs_test' < ./docs/db.sql
@mysql -uroot 'cnpmjs_test' -e 'show tables;'
test: install pretest
@NODE_ENV=test ./node_modules/.bin/mocha \
--harmony-generators \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
--require should \
--require co-mocha \
--require ./test/init.js \
$(MOCHA_OPTS) \
$(TESTS)
test-cov cov: install
@NODE_ENV=test node --harmony \
node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha \
node_modules/.bin/istanbul cover --preserve-comments \
./node_modules/.bin/_mocha \
-- -u exports \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
--require should \
--require co-mocha \
--require ./test/init.js \
$(MOCHA_OPTS) \
$(TESTS)
@-$(MAKE) check-coverage
check-coverage:
@./node_modules/.bin/istanbul check-coverage \
--statements 100 \
--functions 100 \
--branches 100 \
--lines 100
cov:
@./node_modules/.bin/cov coverage
contributors:
contributors: install
@./node_modules/.bin/contributors -f plain -o AUTHORS
autod:
@./node_modules/.bin/autod -w -e public,view,docs,backup
autod: install
@./node_modules/.bin/autod -w -e public,view,docs,backup,coverage
@$(MAKE) install
.PHONY: test

View File

@@ -1,7 +1,7 @@
cnpmjs.org
=======
[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
@@ -9,29 +9,86 @@ cnpmjs.org
## What is this?
Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
Private npm registry and web for Enterprise, base on [koa](http://koajs.com/),
MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
Our goal is to provide a low cost maintenance and easy to use solution for private npm.
## What can you do with `cnpmjs.org`
* Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now)
* Build a mirror NPM. (we use it to build a mirror in China: [cnpmjs.org](http://cnpmjs.org/))
* Build a completely independent NPM registry to store whatever you like.
### Features
* **Simple to deploy**: only need `mysql` and a [simple store system](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
You can get the source code through `npm` or `git`.
* **Low cost and easy maintenance**: `package.json` info store in MySQL, tarball(tgz file) store in CDN or other store systems.
* **Automatic synchronization**: automatic synchronization from any registry specified, support two sync modes:
- Sync all modules from a specified registry, like [npm registry](http://registry.npmjs.org).
- Only sync the modules that exists in your own registry.
* **Manual synchronization**: automatic synchronization may has little delay, but you can syn immediately by manually.
* **Customized client**: we provide a client [cnpm](https://github.com/cnpm/cnpm)
to extend `npm` with more features(`sync` command, [gzip](https://github.com/npm/npm-registry-client/pull/40) support).
And it easy to wrap for your own registry which build with `cnpmjs.org`.
* **Compatible with NPM client**: you can use the origin NPM client with `cnpmjs.org`,
only need to change the registry in config. Even include manual synchronization (through `install` command).
## Getting Start
* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
* install and deploy cnpmjs.org through npm: [examples](https://github.com/cnpm/custom-cnpm-example)
* Mirror NPM in China: [cnpmjs.org](http://cnpmjs.org)
* cnpm client: [cnpm](https://github.com/cnpm/cnpm), `npm install -g cnpm`
* [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
* [NFS guide](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide)
![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
## Develop on your local machine
## Install
### Dependencies
* [node](http://nodejs.org) >=0.11.9
* [mysql](http://dev.mysql.com/downloads/) >= 0.5.0, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
### Start MySQL
```bash
$ npm install --registry=http://r.cnpmjs.org --disturl=http://dist.u.qiniudn.com
$ nohup mysqld &
```
## Usage
### Clone codes and run test
```js
$ node --harmony-generators dispatch.js
```bash
# clone from git
$ git clone https://github.com/cnpm/cnpmjs.org.git
# install dependencies
$ make install
# test
$ make test
# coverage
$ make test-cov
# udpate dependencies
$ make autod
# start server
$ node --harmony_generators dispatch.js
```
## Guide
## How to contribute
* [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
* [NFS guide](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide)
* Clone the project
* Checkout a new branch
* Add new features or fix bugs in the new branch
* Make a pull request and we will review it ASAP
Tips: make sure your code is following the [node-style-guide](https://github.com/felixge/node-style-guide).
## Authors
@@ -39,15 +96,16 @@ $ node --harmony-generators dispatch.js
$ git summary
project : cnpmjs.org
repo age : 3 months
active : 145 days
commits : 366
files : 94
repo age : 4 months ago
commits : 472
active : 167 days
files : 104
authors :
217 fengmk2 59.3%
146 dead_horse 39.9%
2 4simple 0.5%
1 Alsotang 0.3%
272 fengmk2 57.6%
195 dead_horse 41.3%
2 4simple 0.4%
2 Stanley Zheng 0.4%
1 Alsotang 0.2%
```
## License

View File

@@ -15,6 +15,7 @@
* Module dependencies.
*/
var thunkify = require('thunkify-wrap');
var ready = require('ready');
var mysql = require('mysql');
var config = require('../config');
@@ -35,6 +36,10 @@ var pool = mysql.createPool({
exports.pool = pool;
exports.query = function (sql, values, cb) {
if (typeof values === 'function') {
cb = values;
values = null;
}
pool.query(sql, values, function (err, rows) {
cb(err, rows);
});
@@ -59,6 +64,8 @@ exports.escape = function (val) {
ready(exports);
thunkify(exports);
function init() {
exports.query('show tables', function (err, rows) {
if (err) {

View File

@@ -19,6 +19,8 @@ var qn = require('qn');
var config = require('../config');
var client = qn.create(config.qn);
thunkify(client, ['delete', 'uploadFile', 'upload']);
exports._client = client;
/**
@@ -28,38 +30,45 @@ exports._client = client;
* @param {Object} options
* - {String} key
* - {Number} size
* @param {Function(err, result)} callback
* - {Object} result
* - {String} url
*/
exports.upload = function (filepath, options, callback) {
client.delete(options.key, function (err, data) {
client.uploadFile(filepath, {key: options.key, size: options.size}, function (err, data) {
if (err) {
return callback(err);
}
callback(null, {url: data.url});
});
exports.upload = function *(filepath, options) {
try {
yield client.delete(options.key);
} catch (err) {
// ignore error here
}
var res = yield client.uploadFile(filepath, {
key: options.key,
size: options.size
});
var url = res && res[0] ? res[0].url : '';
return { url: url };
};
exports.uploadBuffer = function (buf, options, callback) {
client.delete(options.key, function (err, data) {
client.upload(buf, {key: options.key}, function (err, data) {
if (err) {
return callback(err);
}
callback(null, {url: data.url});
});
});
exports.uploadBuffer = function *(buf, options) {
try {
yield client.delete(options.key);
} catch (err) {
// ignore error here
}
var res = yield client.upload(buf, {key: options.key});
var url = res && res[0] ? res[0].url : '';
return { url: url };
};
exports.url = function (key) {
return client.resourceURL(key);
};
exports.remove = function (key, callback) {
client.delete(key, callback);
exports.remove = function *(key) {
try {
return yield client.delete(key);
} catch (err) {
if (err.name === 'QiniuFileNotExistsError') {
return;
}
throw err;
}
};
thunkify(exports);

38
common/redis.js Normal file
View File

@@ -0,0 +1,38 @@
/**!
* cnpmjs.org - common/redis.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* dead_horse <dead_horse@qq.com> (http://deadhorse.me)
*/
'use strict';
/**
* Module dependencies.
*/
var config = require('../config');
// close redis by set config.redis to `null` or `{}`
if (config.redis && config.redis.host && config.redis.port) {
var redis = require('redis');
var wrapper = require('co-redis');
var logger = require('./logger');
var _client = redis.createClient(config.redis);
_client.on('error', function (err) {
logger.error(err);
});
module.exports = wrapper(_client);
} else {
console.warn('[%s] [worker:%s:common/redis.js] Redis config can not found',
Date(), process.pid);
module.exports = null;
}

View File

@@ -23,6 +23,8 @@ var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 365, signed: fa
var options = {
key: key,
cookie: cookie,
defer: true,
rolling: false
};
if (!config.debug) {

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - config/index.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -17,42 +17,46 @@
var path = require('path');
var fs = require('fs');
var os = require('os');
var mkdirp = require('mkdirp');
var copy = require('copy-to');
fs.existsSync = fs.existsSync || path.existsSync;
var pkg = require('../package.json');
var version = require('../package.json').version;
var root = path.dirname(__dirname);
var config = {
version: pkg.version,
version: version,
registryPort: 7001,
webPort: 7002,
bindingHost: '127.0.0.1', // only binding on 127.0.0.1 for local access
enableCluster: false,
numCPUs: os.cpus().length,
debug: true, // if debug
logdir: path.join(root, '.tmp', 'logs'),
viewCache: false,
// mysql config
mysqlServers: [
{
host: 'keydiary.mysql.rds.aliyuncs.com', // 'db4free.net'
host: '127.0.0.1',
port: 3306,
user: 'cnpmjs',
password: 'cnpmjs123'
user: 'root',
password: ''
}
],
mysqlDatabase: 'cnpmjstest',
mysqlDatabase: 'cnpmjs_test',
mysqlMaxConnections: 4,
mysqlQueryTimeout: 5000,
sessionSecret: 'cnpmjs.org test session secret',
redis: {
host: 'pub-redis-19533.us-east-1-4.3.ec2.garantiadata.com',
port: 19533,
pass: 'cnpmjs_dev'
// host: 'pub-redis-19533.us-east-1-4.3.ec2.garantiadata.com',
// port: 19533,
// pass: 'cnpmjs_dev'
},
jsonLimit: '10mb', // max request json body size
uploadDir: path.join(root, 'public', 'dist'),
uploadDir: path.join(root, '.dist'),
// qiniu cdn: http://www.qiniu.com/, it free for dev.
qn: {
accessKey: "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
@@ -72,8 +76,11 @@ var config = {
debug: false
},
disturl: 'http://dist.u.qiniudn.com',
logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
registryHost: 'r.cnpmjs.org',
// customReadmeFile: __dirname + '/web_readme.md',
customReadmeFile: '', // you can use your custom readme file instead the cnpm one
customFooter: '', // you can add copyright and site total script html here
npmClientName: 'cnpm', // use `${name} install package`
packagePageContributorSearch: true, // package page contributor link to search, default is true
@@ -89,16 +96,24 @@ var config = {
backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
syncModel: 'none', // 'none', 'all', 'exist'
syncConcurrency: 1,
syncInterval: '10m', // sync interval, default is 10 minutes
maxDependencies: 200, // max handle number of package.json `dependencies` property
limit: {
enable: false,
token: 'koa-limit:download',
limit: 1000,
interval: 1000 * 60 * 60 * 24,
whiteList: [],
blackList: [],
message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
}
};
// load config/config.js, everything in config.js will cover the same key in index.js
var customConfig = path.join(root, 'config/config.js');
if (fs.existsSync(customConfig)) {
var options = require(customConfig);
for (var k in options) {
config[k] = options[k];
}
copy(require(customConfig)).override(config);
}
mkdirp.sync(config.logdir);
@@ -110,7 +125,5 @@ config.loadConfig = function (customConfig) {
if (!customConfig) {
return;
}
for (var key in customConfig) {
config[key] = customConfig[key];
}
copy(customConfig).override(config);
};

View File

@@ -18,6 +18,7 @@
var debug = require('debug')('cnpmjs.org:controllers:registry:module');
var path = require('path');
var fs = require('fs');
var util = require('util');
var crypto = require('crypto');
var utility = require('utility');
var coRead = require('co-read');
@@ -31,28 +32,49 @@ var Module = require('../../proxy/module');
var Total = require('../../proxy/total');
var nfs = require('../../common/nfs');
var common = require('../../lib/common');
var Log = require('../../proxy/module_log');
var DownloadTotal = require('../../proxy/download');
var SyncModuleWorker = require('../../proxy/sync_module_worker');
var logger = require('../../common/logger');
var ModuleDeps = require('../../proxy/module_deps');
var ModuleStar = require('../../proxy/module_star');
/**
* show all version of a module
*/
exports.show = function *(next) {
var name = this.params.name;
var modifiedTime = yield *Module.getLastModified(name);
debug('show %s, last modified: %s', name, modifiedTime);
if (modifiedTime) {
// use modifiedTime as etag
this.set('ETag', '"' + modifiedTime.getTime() + '"');
var r = yield [Module.listTags(name), Module.listByName(name)];
// must set status first
this.status = 200;
if (this.fresh) {
debug('%s not change at %s, 304 return', name, modifiedTime);
this.status = 304;
return;
}
}
var r = yield [
Module.listTags(name),
Module.listByName(name),
ModuleStar.listUsers(name)
];
var tags = r[0];
var rows = r[1];
debug('show module, user: %s, allowSync: %s, isAdmin: %s',
this.session.name, this.session.allowSync, this.session.isAdmin);
var users = r[2];
var userMap = {};
for (var i = 0; i < users.length; i++) {
userMap[users[i]] = true;
}
users = userMap;
// if module not exist in this registry,
// sync the module backend and return package info from official registry
if (rows.length === 0) {
if (!this.session.allowSync) {
if (!this.allowSync) {
this.status = 404;
this.body = {
error: 'not_found',
@@ -60,9 +82,8 @@ exports.show = function *(next) {
};
return;
}
var username = (this.session && this.session.name) || 'anonymous';
var result = yield SyncModuleWorker.sync(name, username);
this.status = result.ok ? 200 : result.statusCode;
var result = yield SyncModuleWorker.sync(name, 'sync-by-install');
this.status = result.ok ? 200 : (result.statusCode || 500);
this.body = result.pkg;
return;
}
@@ -81,6 +102,7 @@ exports.show = function *(next) {
var versions = {};
var times = {};
var attachments = {};
var createdTime = null;
for (var i = 0; i < rows.length; i++) {
var row = rows[i];
if (row.version === 'next') {
@@ -91,12 +113,27 @@ exports.show = function *(next) {
common.setDownloadURL(pkg, this);
pkg._cnpm_publish_time = row.publish_time;
versions[pkg.version] = pkg;
times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
var t = times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
if ((!distTags.latest && !latestMod) || distTags.latest === row.version) {
latestMod = row;
readme = pkg.readme;
}
delete pkg.readme;
if (!createdTime || t < createdTime) {
createdTime = t;
}
}
if (modifiedTime && createdTime) {
var ts = {
modified: modifiedTime,
created: createdTime,
};
for (var t in times) {
ts[t] = times[t];
}
times = ts;
}
if (!latestMod) {
@@ -112,23 +149,30 @@ exports.show = function *(next) {
rev = String(nextMod.id);
}
var pkg = latestMod.package;
var info = {
_id: name,
_rev: rev,
name: name,
description: latestMod.package.description,
description: pkg.description,
"dist-tags": distTags,
maintainers: latestMod.package.maintainers,
maintainers: pkg.maintainers,
time: times,
author: latestMod.package.author,
repository: latestMod.package.repository,
users: users,
author: pkg.author,
repository: pkg.repository,
versions: versions,
readme: readme,
_attachments: attachments,
};
debug('show module %s: %s, latest: %s', name, rev, latestMod.version);
info.readmeFilename = pkg.readmeFilename;
info.homepage = pkg.homepage;
info.bugs = pkg.bugs;
info.license = pkg.license;
debug('show module %s: %s, latest: %s', name, rev, latestMod.version);
this.body = info;
};
@@ -149,9 +193,8 @@ exports.get = function *(next) {
this.body = mod.package;
return;
}
// if not fond, sync from source registry
if (!this.session.allowSync) {
if (!this.allowSync) {
this.status = 404;
this.body = {
error: 'not exist',
@@ -160,8 +203,7 @@ exports.get = function *(next) {
return;
}
var username = (this.session && this.session.username) || 'anonymous';
var result = yield SyncModuleWorker.sync(name, username);
var result = yield SyncModuleWorker.sync(name, 'sync-by-install');
var pkg = result.pkg && result.pkg.versions[version];
if (!pkg) {
this.status = 404;
@@ -192,7 +234,7 @@ exports.download = function *(next) {
if (!row || !row.package || !row.package.dist) {
if (!url) {
return yield next;
return yield* next;
}
this.status = 302;
this.set('Location', url);
@@ -211,7 +253,7 @@ exports.download = function *(next) {
// else use `dist.key` to get tarball from nfs
if (!nfs.download) {
return yield next;
return yield* next;
}
_downloads[name] = (_downloads[name] || 0) + 1;
@@ -289,10 +331,9 @@ exports.upload = function *(next) {
var length = Number(this.get('content-length')) || 0;
if (!length || !this.is('application/octet-stream')) {
debug('request length or type error');
return yield next;
return yield *next;
}
var username = this.session.name;
var username = this.user.name;
var name = this.params.name;
var id = Number(this.params.rev);
var filename = this.params.filename;
@@ -304,9 +345,9 @@ exports.upload = function *(next) {
var mod = yield Module.getById(id);
if (!mod) {
debug('can not get this module');
return yield next;
return yield* next;
}
if (!common.isMaintainer(this, mod.package.maintainers) || mod.name !== name) {
if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
this.status = 403;
this.body = {
error: 'no_perms',
@@ -330,7 +371,7 @@ exports.upload = function *(next) {
var dataSize = 0;
var buf;
while(buf = yield coRead(this.req)) {
while (buf = yield coRead(this.req)) {
shasum.update(buf);
dataSize += buf.length;
yield coWrite(ws, buf);
@@ -397,7 +438,7 @@ function _addDepsRelations(pkg) {
}
exports.updateLatest = function *(next) {
var username = this.session.name;
var username = this.user.name;
var name = this.params.name;
var version = semver.valid(this.params.version);
if (!version) {
@@ -412,12 +453,9 @@ exports.updateLatest = function *(next) {
var nextMod = yield Module.get(name, 'next');
if (!nextMod) {
debug('can not get nextMod');
return yield next;
return yield* next;
}
var match = nextMod.package.maintainers.filter(function (item) {
return item.name === username;
});
if (match.length === 0) {
if (!common.isMaintainer(this.user, nextMod.package.maintainers)) {
this.status = 401;
this.body = {
error: 'noperms',
@@ -479,7 +517,7 @@ exports.addPackageAndDist = function *(next) {
// length: 9883
var pkg = this.request.body;
var username = this.session.name;
var username = this.user.name;
var name = this.params.name;
var filename = Object.keys(pkg._attachments || {})[0];
var version = Object.keys(pkg.versions || {})[0];
@@ -580,11 +618,11 @@ exports.addPackageAndDist = function *(next) {
};
exports.add = function *(next) {
var username = this.session.name;
var username = this.user.name;
var name = this.params.name;
var pkg = this.request.body || {};
if (!common.isMaintainer(this, pkg.maintainers)) {
if (!common.isMaintainer(this.user, pkg.maintainers)) {
this.status = 403;
this.body = {
error: 'no_perms',
@@ -624,7 +662,7 @@ exports.add = function *(next) {
var maintainers = latestMod && latestMod.package.maintainers.length > 0 ?
latestMod.package.maintainers : nextMod.package.maintainers;
if (!common.isMaintainer(this, maintainers)) {
if (!common.isMaintainer(this.user, maintainers)) {
this.status = 403;
this.body = {
error: 'no_perms',
@@ -651,21 +689,65 @@ exports.add = function *(next) {
};
};
exports.updateOrRemove = function *(next) {
debug('updateOrRemove module %s, %j', this.params.name, this.request.body);
var body = this.request.body;
if (body.versions) {
yield *exports.removeWithVersions.call(this, next);
} else if (body.maintainers && body.maintainers.length > 0) {
yield *exports.updateMaintainers.call(this, next);
} else {
yield *next;
}
};
exports.updateMaintainers = function *(next) {
var name = this.params.name;
var body = this.request.body;
debug('updateMaintainers module %s, %j', name, body);
var latestMod = yield Module.getLatest(name);
if (!latestMod || !latestMod.package) {
return yield *next;
}
if (!common.isMaintainer(this.user, latestMod.package.maintainers)) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not publish this module'
};
return;
}
var r = yield *Module.updateMaintainers(latestMod.id, body.maintainers);
debug('result: %j', r);
this.status = 201;
this.body = {
ok: true,
id: name,
rev: String(latestMod.id),
};
};
exports.removeWithVersions = function *(next) {
debug('removeWithVersions module %s, with info %j', this.params.name, this.request.body);
var username = this.user.name;
var name = this.params.name;
var username = this.session.name;
var versions = this.request.body.versions || {};
debug('removeWithVersions module %s, with versions %j', name, Object.keys(versions));
// step1: list all the versions
var mods = yield Module.listByName(name);
if (!mods || !mods.length) {
return yield next;
return yield *next;
}
// step2: check permission
var firstMod = mods[0];
if (!common.isMaintainer(this, firstMod.package.maintainers) || firstMod.name !== name) {
if (!common.isMaintainer(this.user, firstMod.package.maintainers) || firstMod.name !== name) {
this.status = 403;
this.body = {
error: 'no_perms',
@@ -730,23 +812,22 @@ exports.removeWithVersions = function *(next) {
debug('no tag need to be remove');
}
this.status = 201;
this.bdoy = { ok: true };
this.body = { ok: true };
};
exports.removeTar = function *(next) {
debug('remove tarball with filename: %s, id: %s', this.params.filename, this.params.rev);
var id = Number(this.params.rev);
var filename = this.params.filename;
var name = this.params.name;
var username = this.session.name;
var username = this.user.name;
var mod = yield Module.getById(id);
if (!mod) {
return yield next;
return yield* next;
}
if (!common.isMaintainer(this, mod.package.maintainers) || mod.name !== name) {
if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
this.status = 403;
this.body = {
error: 'no_perms',
@@ -762,20 +843,19 @@ exports.removeTar = function *(next) {
exports.removeAll = function *(next) {
debug('remove all the module with name: %s, id: %s', this.params.name, this.params.rev);
var id = Number(this.params.rev);
// var id = Number(this.params.rev);
var name = this.params.name;
var username = this.session.name;
var mods = yield Module.listByName(name);
debug('removeAll module %s: %d', name, mods.length);
var mod = mods[0];
if (!mod) {
return yield next;
return yield* next;
}
if (!common.isMaintainer(this, mod.package.maintainers) || mod.name !== name) {
res.status = 403;
res.body = {
if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
this.status = 403;
this.body = {
error: 'no_perms',
reason: 'Current user can not delete this tarball'
};
@@ -821,13 +901,17 @@ function parseModsForList(updated, mods, ctx) {
return results;
}
exports.listAllModules = function *(next) {
exports.listAllModules = function *() {
var updated = Date.now();
var mods = yield Module.listSince(0);
this.body = parseModsForList(updated, mods, this);
var mods = yield Module.listAllNames();
var result = { _updated: updated };
mods.forEach(function (mod) {
result[mod.name] = true;
});
this.body = result;
};
exports.listAllModulesSince = function *(next) {
exports.listAllModulesSince = function *() {
var query = this.query || {};
if (query.stale !== 'update_after') {
this.status = 400;
@@ -842,11 +926,70 @@ exports.listAllModulesSince = function *(next) {
var startkey = Number(query.startkey) || 0;
var updated = Date.now();
var mods = yield Module.listSince(startkey);
this.body = parseModsForList(updated, mods, this);
var result = { _updated: updated };
mods.forEach(function (mod) {
result[mod.name] = true;
});
this.body = result;
};
exports.listAllModuleNames = function *(next) {
exports.listAllModuleNames = function *() {
this.body = (yield Module.listShort()).map(function (m) {
return m.name;
});
};
exports.updateTag = function *() {
var version = this.request.body;
var tag = this.params.tag;
var name = this.params.name;
if (!version) {
this.status = 400;
this.body = {
error: 'version_missed',
reason: 'version not found'
};
return;
}
if (!semver.valid(version)) {
this.status = 403;
var reason = util.format('setting tag %s to invalid version: %s: %s/%s',
tag, version, name, tag);
this.body = {
error: 'forbidden',
reason: reason
};
return;
}
var mod = yield Module.get(name, version);
if (!mod) {
this.status = 403;
var reason = util.format('setting tag %s to unknown version: %s: %s/%s',
tag, version, name, tag);
this.body = {
error: 'forbidden',
reason: reason
};
return;
}
// check permission
if (!common.isMaintainer(this.user, mod.package.maintainers)) {
this.status = 403;
this.body = {
error: 'forbidden',
reason: 'no permission to modify ' + name
};
return;
}
yield Module.addTag(name, tag, version);
this.status = 201;
this.body = {
ok: true
};
};

View File

@@ -15,38 +15,83 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:controllers:registry');
var logger = require('../../common/logger');
var debug = require('debug')('cnpmjs.org:controllers:registry:user');
var utility = require('utility');
var crypto = require('crypto');
var User = require('../../proxy/user');
var config = require('../../config');
exports.show = function *(next) {
var name = this.params.name;
var user = yield User.get(name);
if (!user) {
return yield next;
return yield *next;
}
this.etag = '"' + user.rev + '"';
var data = {
_id: 'org.couchdb.user:' + user.name,
_rev: user.rev,
name: user.name,
email: user.email,
type: 'user',
roles: [],
date: user.gmt_modified,
var data = user.json;
if (!data) {
data = {
_id: 'org.couchdb.user:' + user.name,
_rev: user.rev,
name: user.name,
email: user.email,
type: 'user',
roles: [],
date: user.gmt_modified,
};
}
data._cnpm_meta = {
id: user.id,
npm_user: user.npm_user,
gmt_create: user.gmt_create,
gmt_modified: user.gmt_modified,
admin: !!config.admins[user.name],
};
this.body = data;
};
function ensurePasswordSalt(user, body) {
if (!user.password_sha && body.password) {
// create password_sha on server
user.salt = crypto.randomBytes(30).toString('hex');
user.password_sha = utility.sha1(body.password + user.salt);
}
}
// npm 1.4.4
// add new user first
// @see https://github.com/npm/npm-registry-client/commit/effb4bc88d443f764f2c2e8b4dd583cc72cf6084
// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
// 'accept-encoding': 'gzip',
// 'user-agent': 'node/v0.11.12 darwin x64',
// host: '127.0.0.1:7001',
// 'content-type': 'application/json',
// 'content-length': '150',
// connection: 'close' } { name: 'mk2',
// password: '123456',
// email: 'fengmk2@gmail.com',
// _id: 'org.couchdb.user:mk2',
// type: 'user',
// roles: [],
// date: '2014-03-15T02:33:19.465Z' }
// old npm flow
// json:
// { name: 'fengmk2',
// salt: 'xxxx',
// password_sha: 'xxxxxx',
// email: 'fengmk2@gmail.com',
// _id: 'org.couchdb.user:fengmk2',
// type: 'user',
// roles: [],
// date: '2013-12-04T12:56:13.714Z' } }
// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
// 'user-agent': 'node/v0.8.26 darwin x64',
// host: '127.0.0.1:7001',
// 'content-type': 'application/json',
// 'content-length': '258',
// connection: 'keep-alive' }
// { name: 'mk2',
// salt: '18d8d51936478446a5466d4fb1633b80f3838b4caaa03649a885ac722cd6',
// password_sha: '8f4408912a6db1d96b132a90856d99db029cef3d',
// email: 'fengmk2@gmail.com',
// _id: 'org.couchdb.user:mk2',
// type: 'user',
// roles: [],
// date: '2014-03-15T02:39:25.696Z' }
exports.add = function *() {
var name = this.params.name;
var body = this.request.body || {};
@@ -59,11 +104,13 @@ exports.add = function *() {
// roles: body.roles || [],
};
ensurePasswordSalt(user, body);
if (!user.name || !user.salt || !user.password_sha || !user.email) {
this.status = 422;
this.body = {
error: 'paramError',
reason: 'params missing'
reason: 'params missing, name, email or password missing.'
};
return;
}
@@ -74,7 +121,7 @@ exports.add = function *() {
this.status = 409;
this.body = {
error: 'conflict',
reason: 'Document update conflict.'
reason: 'User ' + name + ' already exists.'
};
return;
}
@@ -102,8 +149,8 @@ exports.authSession = function *() {
this.body = {ok: false, name: null, roles: []};
return;
}
this.session.name = user.name;
var session = yield *this.session;
session.name = user.name;
this.body = {ok: true, name: user.name, roles: []};
};
@@ -111,13 +158,12 @@ exports.update = function *(next) {
var name = this.params.name;
var rev = this.params.rev;
if (!name || !rev) {
return yield next;
return yield* next;
}
debug('update: %s, rev: %s, user.name: %s', name, rev, this.user.name);
debug('update: %s, rev: %s, session.name: %s', name, rev, this.session.name);
if (name !== this.session.name) {
// must authSession first
if (name !== this.user.name) {
// must auth user first
this.status = 401;
this.body = {
error: 'unauthorized',
@@ -136,6 +182,18 @@ exports.update = function *(next) {
rev: body.rev || body._rev,
// roles: body.roles || [],
};
ensurePasswordSalt(user, body);
if (!user.name || !user.salt || !user.password_sha || !user.email) {
this.status = 422;
this.body = {
error: 'paramError',
reason: 'params missing, name, email or password missing.'
};
return;
}
var result = yield User.update(user);
if (!result) {
this.status = 409;

View File

@@ -18,11 +18,11 @@ var Log = require('../proxy/module_log');
var SyncModuleWorker = require('../proxy/sync_module_worker');
exports.sync = function *() {
var username = this.session.name || 'anonymous';
var username = this.user.name || 'anonymous';
var name = this.params.name;
var publish = this.query.publish === 'true';
var noDep = this.query.nodeps === 'true';
if (publish && !this.session.isAdmin) {
if (publish && !this.user.isAdmin) {
this.status = 403;
this.body = {
error: 'no_perms',
@@ -39,7 +39,7 @@ exports.sync = function *() {
var result = yield SyncModuleWorker.sync(name, username, options);
// friendly 404 reason info
if (result.staticCache === 404) {
if (result.statusCode === 404) {
this.status = 404;
this.body = {
ok: false,
@@ -48,7 +48,7 @@ exports.sync = function *() {
return;
}
if (!result.ok) {
this.status = result.statusCode;
this.status = result.statusCode || 500;
this.body = result.pkg;
return;
}
@@ -61,11 +61,10 @@ exports.sync = function *() {
exports.getSyncLog = function *(next) {
var logId = this.params.id;
var name = this.params.name;
var offset = Number(this.query.offset) || 0;
var row = yield Log.get(logId);
if (!row) {
return yield next;
return yield* next;
}
var log = row.log.trim();

23
controllers/web/dist.js Normal file
View File

@@ -0,0 +1,23 @@
/**!
* cnpmjs.org - controllers/web/dist.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
"use strict";
/**
* Module dependencies.
*/
var config = require('../../config');
exports.redirect = function *(next) {
var params = this.params;
var url = config.disturl + (params[0] || '/');
this.redirect(url);
};

View File

@@ -14,6 +14,7 @@
* Module dependencies.
*/
var bytes = require('bytes');
var giturl = require('giturl');
var moment = require('moment');
var eventproxy = require('eventproxy');
@@ -28,6 +29,7 @@ var sync = require('../sync');
var Log = require('../../proxy/module_log');
var ModuleDeps = require('../../proxy/module_deps');
var setDownloadURL = require('../../lib/common').setDownloadURL;
var ModuleStar = require('../../proxy/module_star');
exports.display = function *(next) {
var params = this.params;
@@ -47,20 +49,23 @@ exports.display = function *(next) {
var r = yield [
Module[getPackageMethod].apply(Module, getPackageArgs),
down.total(name),
ModuleDeps.list(name)
ModuleDeps.list(name),
ModuleStar.listUsers(name),
];
var pkg = r[0];
var download = r[1];
var dependents = (r[2] || []).map(function (item) {
return item.deps;
});
var users = r[3];
if (!pkg || !pkg.package) {
return yield next;
return yield* next;
}
pkg.package.fromNow = moment(pkg.publish_time).fromNow();
pkg = pkg.package;
pkg.users = users;
pkg.readme = marked(pkg.readme || '');
if (!pkg.readme) {
pkg.readme = pkg.description || '';
@@ -104,6 +109,10 @@ exports.display = function *(next) {
pkg.dependents = dependents;
if (pkg.dist) {
pkg.dist.size = bytes(pkg.dist.size || 0);
}
yield this.render('package', {
title: 'Package - ' + pkg.name,
package: pkg,
@@ -115,20 +124,31 @@ exports.search = function *(next) {
var params = this.params;
var word = params.word;
var result = yield Module.search(word);
var match = null;
for (var i = 0; i < result.searchMatchs.length; i++) {
var p = result.searchMatchs[i];
if (p.name === word) {
match = p;
break;
}
}
// return a json result
if (this.query && this.query.type === 'json') {
this.body = {
keyword: word,
match: match,
packages: result.searchMatchs,
keywords: result.keywordMatchs
keywords: result.keywordMatchs,
};
this.charset = 'utf-8';
this.type = 'application/json; charset=utf-8';
return;
}
yield this.render('search', {
title: 'Keyword - ' + word,
keyword: word,
match: match,
packages: result.searchMatchs,
keywords: result.keywordMatchs,
});

View File

@@ -23,7 +23,7 @@ exports.display = function *(next) {
var packages = r[0];
var user = r[1];
if (!user && !packages.length) {
return yield next;
return yield* next;
}
user = {
name: name,

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - dispatch.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -17,7 +17,6 @@
var path = require('path');
var util = require('util');
var fs = require('fs');
var cluster = require('cluster');
var config = require('./config');
var workerPath = path.join(__dirname, 'worker.js');
@@ -30,25 +29,25 @@ if (config.enableCluster) {
});
cluster.on('fork', function (worker) {
console.log('[%s] [worker:%d] new worker start', new Date(), worker.process.pid);
console.log('[%s] [worker:%d] new worker start', Date(), worker.process.pid);
});
cluster.on('disconnect', function (worker) {
var w = cluster.fork();
console.error('[%s] [master:%s] wroker:%s disconnect! new worker:%s fork',
new Date(), process.pid, worker.process.pid, w.process.pid);
console.error('[%s] [master:%s] wroker:%s disconnect, suicide: %s, state: %s. New worker:%s fork',
Date(), process.pid, worker.process.pid, worker.suicide, worker.state, w.process.pid);
});
cluster.on('exit', function (worker, code, signal) {
var exitCode = worker.process.exitCode;
var err = new Error(util.format('worker %s died (code: %s, signal: %s)', worker.process.pid, exitCode, signal));
var err = new Error(util.format('worker %s died (code: %s, signal: %s, suicide: %s, state: %s)',
worker.process.pid, exitCode, signal, worker.suicide, worker.state));
err.name = 'WorkerDiedError';
console.error(err);
console.error('[%s] [master:%s] wroker exit: %s', Date(), process.pid, err.stack);
});
var numCPUs = require('os').cpus().length;
// Fork workers.
for (var i = 0; i < numCPUs; i++) {
for (var i = 0; i < config.numCPUs; i++) {
cluster.fork();
}

View File

@@ -9,10 +9,15 @@ CREATE TABLE `user` (
`roles` varchar(200) NOT NULL DEFAULT '[]',
`rev` varchar(40) NOT NULL,
`email` varchar(400) NOT NULL,
`json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
`npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false',
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='user base info';
-- ALTER TABLE `user`
-- ADD `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
-- ADD `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false';
CREATE TABLE `module_keyword` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
@@ -25,6 +30,16 @@ CREATE TABLE `module_keyword` (
KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
CREATE TABLE `module_star` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
PRIMARY KEY (`id`),
UNIQUE KEY `user_module_name` (`user`,`name`),
KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
CREATE TABLE `module` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',

View File

@@ -1,5 +1,24 @@
# npm publish flow
## Flows
1. try to put package.json and tgz, maybe base64 tgz body
2. if new version not exists, publish success
3. if new version exists, 409, try to get full package info with ?write=true
4. if new version had publish, show: "Update the 'version' field in package.json and try again."
```bash
$ cnpm publish
npm http PUT http://r.cnpmjs.org/cnpmjs.org
npm http 409 http://r.cnpmjs.org/cnpmjs.org
npm http GET http://r.cnpmjs.org/cnpmjs.org?write=true
npm http 200 http://r.cnpmjs.org/cnpmjs.org?write=true
npm ERR! publish fail Cannot publish over existing version.
npm ERR! publish fail Update the 'version' field in package.json and try again.
```
## Details
code: https://github.com/isaacs/npm-registry-client/blob/master/lib/publish.js
* couch login if token not exists: [couch-login](https://github.com/isaacs/couch-login)

View File

@@ -103,13 +103,13 @@ npm install -g cnpm
alias lnpm='cnpm --registry=http://localhost:7001\
--registryweb=http://localhost:7002\
--cache=$HOME/.npm/.cache/lnpm\
--disturl=http://dist.u.qiniudn.com\
--disturl=http://cnpmjs.org/dist\
--userconfig=$HOME/.lnpmrc'
#or put this in .zshrc or .bashrc
echo "#lnpm alias\nalias lnpm='cnpm --registry=http://localhost:7001\
--registryweb=http://localhost:7002\
--cache=$HOME/.npm/.cache/lnpm\
--disturl=http://dist.u.qiniudn.com\
--disturl=http://cnpmjs.org/dist\
--userconfig=$HOME/.lnpmrc'" >> $HOME/.zshrc && source $HOME/.zshrc
```

View File

@@ -1,18 +1,6 @@
# cnpmjs.org: Private npm registry and web for Enterprise
# cnpmjs.org: Private npm registry and web for Company
[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
## What is this?
> Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
## Install your private npm registry
@see [Install and Get Started](/install).
So `cnpm` is meaning: **Company npm**.
## Registry
@@ -114,29 +102,29 @@ $(function () {
});
</script>
## cnpm cli
## Usage
alias it:
use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
```
npm install -g cnpm --registry=http://r.cnpmjs.org
```
Or you can alias NPM to use it:
```bash
alias cnpm="npm --registry=http://r.cnpmjs.org \
--cache=$HOME/.npm/.cache/cnpm \
--disturl=http://dist.u.qiniudn.com \
--disturl=http://dist.cnpmjs.org \
--userconfig=$HOME/.cnpmrc"
#Or alias it in .bashrc or .zshrc
$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
--cache=$HOME/.npm/.cache/cnpm \
--disturl=http://dist.u.qiniudn.com \
--disturl=http://dist.cnpmjs.org \
--userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
```
Or you can just use our `cnpm` cli:
```bash
$ npm install cnpm -g
```
### install
Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
@@ -180,33 +168,15 @@ $ cnpm info cnpm
@see Github [Issues](https://github.com/cnpm/cnpmjs.org/issues)
## Authors
## Histories
Release [History](/history).
```bash
$ git summary
project : cnpmjs.org
repo age : 7 weeks
active : 132 days
commits : 315
files : 88
authors :
190 fengmk2 60.3%
122 dead_horse 38.7%
2 4simple 0.6%
1 Alsotang 0.3%
```
## npm and cnpm relation
![npm&cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=383&h=284)
## 捐赠 Donate
如果您觉得 [cnpmjs.org] 对您有帮助,欢迎请作者一杯咖啡.
如果您觉得 [cnpmjs.org](/) 对您有帮助,欢迎请作者一杯咖啡.
[![Donate](https://img.alipay.com/sys/personalprod/style/mc/btn-index.png)](https://me.alipay.com/imk2)
[cnpmjs.org]: http://cnpmjs.org/
[registry.cnpmjs.org]: http://registry.cnpmjs.org/

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - index.js
*
* Copyright(c) cnpmjs.org and other contributors.

View File

@@ -31,7 +31,7 @@ exports.getCDNKey = function (name, filename) {
exports.setDownloadURL = function (pkg, ctx, host) {
if (pkg.dist) {
host = host || ctx.get('host') || ctx.host;
host = host || ctx.host;
pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
ctx.protocol,
host, pkg.name, pkg.name, pkg.version);
@@ -42,12 +42,12 @@ exports.isAdmin = function (username) {
return typeof config.admins[username] === 'string';
};
exports.isMaintainer = function (ctx, maintainers) {
if (ctx.session.isAdmin) {
exports.isMaintainer = function (user, maintainers) {
if (user.isAdmin) {
return true;
}
var username = ctx.session.name;
var username = user.name;
maintainers = maintainers || [];
var match = maintainers.filter(function (item) {
return item.name === username;

View File

@@ -21,28 +21,26 @@ var common = require('../lib/common');
module.exports = function (options) {
return function *auth(next) {
debug('%s, %s, %j', this.url, this.sessionId, this.session);
if (!this.session) {
// redis crash
this.session = {};
return yield next;
}
this.session.onlySync = config.enablePrivate ? true : false;
if (this.session.name) {
this.session.isAdmin = common.isAdmin(this.session.name);
debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
this.session.name, this.session.onlySync, this.session.isAdmin, this.header);
return yield next;
var session = yield *this.session;
debug('%s, %s, %j', this.url, this.sessionId, session);
this.user = {};
if (session.name) {
this.user.name = session.name;
this.user.isAdmin = common.isAdmin(session.name);
debug('auth exists user: %j, headers: %j', this.user, this.header);
return yield *next;
}
var authorization = (this.get('authorization') || '').split(' ')[1] || '';
authorization = authorization.trim();
if (!authorization) {
return yield next;
return yield *next;
}
authorization = new Buffer(authorization, 'base64').toString().split(':');
if (authorization.length !== 2) {
return yield next;
return yield *next;
}
var username = authorization[0];
@@ -51,15 +49,12 @@ module.exports = function (options) {
var row = yield User.auth(username, password);
if (!row) {
debug('auth fail user: %j, headers: %j', row, this.header);
this.session.name = null;
this.session.isAdmin = false;
return yield next;
return yield *next;
}
this.session.name = row.name;
this.session.isAdmin = common.isAdmin(this.session.name);
debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
row, this.session.onlySync, this.session.isAdmin, this.header);
yield next;
this.user.name = row.name;
this.user.isAdmin = common.isAdmin(row.name);
debug('auth pass user: %j, headers: %j', this.user, this.header);
yield *next;
};
};

34
middleware/limit.js Normal file
View File

@@ -0,0 +1,34 @@
/**!
* cnpmjs.org - middleware/limit.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* dead_horse <dead_horse@qq.com> (http://deadhorse.me)
*/
'use strict';
/**
* Module dependencies.
*/
var config = require('../config');
var limit = require('koa-limit');
var store = require('../common/redis');
var limitConfig = config.limit;
if (!limitConfig.enable) {
module.exports = function *ignoreLimit(next) {
yield *next;
};
} else {
if (!config.debug) {
limitConfig.store = store;
}
module.exports = limit(limitConfig);
}

View File

@@ -15,7 +15,7 @@
*/
module.exports = function *login(next) {
if (!this.session.name) {
if (!this.user.name) {
this.status = 401;
this.body = {
error: 'unauthorized',
@@ -23,5 +23,5 @@ module.exports = function *login(next) {
};
return;
}
yield next;
yield *next;
};

View File

@@ -30,5 +30,5 @@ module.exports = function *publishable(next) {
this.charset = 'utf-8';
this.body = template.replace('${host}', this.host);
}
yield next;
yield *next;
};

View File

@@ -14,8 +14,10 @@
* Module dependencies.
*/
var config = require('../config');
module.exports = function *publishable(next) {
if (this.session.onlySync && !this.session.isAdmin) {
if (config.enablePrivate && !this.user.isAdmin) {
// private mode, only admin user can publish
this.status = 403;
this.body = {
@@ -24,5 +26,5 @@ module.exports = function *publishable(next) {
};
return;
}
yield next;
yield *next;
};

View File

@@ -15,7 +15,7 @@
*/
module.exports = function *notFound(next) {
yield next;
yield *next;
if (this.status) {
return;

29
middleware/static.js Normal file
View File

@@ -0,0 +1,29 @@
/**!
* cnpmjs.org - middleware/static.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var path = require('path');
var middlewares = require('koa-middlewares');
var config = require('../config');
var staticDir = path.join(path.dirname(__dirname), 'public');
module.exports = middlewares.staticCache(staticDir, {
buffer: config.debug ? false : true,
maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
alas: {
'/favicon.ico': '/favicon.png'
}
});

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - middleware/sync_by_install.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -14,32 +14,27 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:middleware:sync_by_install');
var config = require('../config');
/**
* req.session.allowSync - allow sync triggle by cnpm install
* this.allowSync - allow sync triggle by cnpm install
*/
module.exports = function *(next) {
module.exports = function *syncByInstall(next) {
if (!config.syncByInstall || !config.enablePrivate) {
// only config.enablePrivate should enable sync on install
return yield next;
return yield *next;
}
// request not by node, consider it request from web
if (this.get('user-agent') && this.get('user-agent').indexOf('node') !== 0) {
return yield next;
var ua = this.get('user-agent');
if (!ua || ua.indexOf('node') < 0) {
return yield *next;
}
this.session.allowSync = true;
if (this.session.isAdmin) {
// if current user is admin, should not enable auto sync on install, because it would be unpublish
this.session.allowSync = false;
if (this.query.write) {
return yield *next;
}
// TODO: allow sync will let publish sync package...
this.session.allowSync = false;
debug('%s allowSync: %s', this.session.name, this.session.allowSync);
yield next;
this.allowSync = true;
yield *next;
};

View File

@@ -15,11 +15,17 @@
*/
module.exports = function *notFound(next) {
yield next;
yield *next;
if (this.status) {
return;
}
var m = /^\/([\w\-\_\.]+)\/?$/.exec(this.url);
if (m) {
return this.redirect('/package/' + m[1]);
}
this.status = 404;
this.type = 'text/html';
this.charset = 'utf-8';

View File

@@ -1,58 +1,64 @@
{
"name": "cnpmjs.org",
"version": "0.3.4",
"version": "0.4.3",
"description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
"main": "index.js",
"scripts": {
"test": "make test",
"test": "make install && make jshint && make test",
"start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
"status": "./bin/nodejsctl status",
"stop": "./bin/nodejsctl stop"
},
"config": {
"cov": {
"threshold": 83
}
},
"dependencies": {
"co-read": "0.0.1",
"bytes": "0.3.0",
"co": "3.0.5",
"co-defer": "0.1.0",
"co-gather": "0.0.1",
"co-read": "0.0.2",
"co-redis": "1.1.0",
"co-urllib": "0.2.1",
"co-write": "0.3.0",
"debug": "0.7.4",
"eventproxy": "0.3.0",
"forward": "0.0.4",
"copy-to": "0.0.3",
"debug": "0.8.0",
"eventproxy": "0.3.1",
"giturl": "0.0.2",
"graceful": "0.0.6",
"gravatar": "1.0.6",
"humanize-number": "0.0.2",
"koa": "0.5.0",
"koa-markdown": "0.0.2",
"koa-middlewares": "0.0.8",
"koa": "0.5.5",
"koa-limit": "1.0.0",
"koa-markdown": "0.0.3",
"koa-middlewares": "0.1.3",
"logfilestream": "0.1.0",
"marked": "0.3.1",
"marked": "0.3.2",
"microtime": "0.5.1",
"mime": "1.2.11",
"mkdirp": "0.3.5",
"moment": "2.5.1",
"moment": "2.6.0",
"ms": "0.6.2",
"mysql": "2.1.0",
"nodemailer": "0.6.1",
"qn": "0.2.0",
"multiline": "0.3.2",
"mysql": "2.1.1",
"nodemailer": "0.6.3",
"qn": "0.2.1",
"ready": "0.1.1",
"redis": "0.10.1",
"semver": "2.2.1",
"thunkify-wrap": "0.0.3",
"urllib": "0.5.5",
"utility": "0.1.10"
"thunkify-wrap": "0.1.1",
"utility": "0.1.12"
},
"devDependencies": {
"autod": ">=0.0.13",
"chunkstream": "0.0.1",
"co-mocha": "0.0.2",
"contributors": "*",
"cov": "*",
"istanbul": "git://github.com/gotwarlost/istanbul.git#harmony",
"mm": "0.2.0",
"istanbul-harmony": "*",
"jshint": "*",
"mm": "0.2.1",
"mocha": "*",
"pedding": "0.0.3",
"should": "3.1.3",
"supertest": "0.9.0"
"should": "3.3.1",
"supertest": "0.11.0"
},
"homepage": "https://github.com/cnpm/cnpmjs.org",
"repository": {
@@ -65,7 +71,11 @@
"email": "fengmk2@gmail.com"
},
"keywords": [
"cnpmjs.org", "npm", "npmjs", "npmjs.org", "registry"
"cnpmjs.org",
"npm",
"npmjs",
"npmjs.org",
"registry"
],
"engines": {
"node": ">= 0.11.9"

View File

@@ -17,28 +17,44 @@
var thunkify = require('thunkify-wrap');
var config = require('../config');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var PLUS_SQL = 'INSERT INTO download_total(gmt_create, gmt_modified, \
date, name, count) \
VALUES(now(), now(), ?, ?, ?) \
ON DUPLICATE KEY UPDATE gmt_modified=now(), \
count=count + VALUES(count), name=VALUES(name), date=VALUES(date);';
var PLUS_SQL = multiline(function () {;/*
INSERT INTO
download_total(gmt_create, gmt_modified, date, name, count)
VALUES
(now(), now(), ?, ?, ?)
ON DUPLICATE KEY UPDATE
count=count + VALUES(count),
name=VALUES(name),
date=VALUES(date);
*/});
exports.plusTotal = function (data, callback) {
mysql.query(PLUS_SQL, [data.date, data.name, data.count], callback);
};
var SELECT_ONE_TOTAL_SQL = 'SELECT date, count FROM download_total WHERE date>=? AND date<=? AND name=?;';
var SELECT_ONE_TOTAL_SQL = multiline(function () {;/*
SELECT
date, count
FROM
download_total
WHERE
date>=? AND date<=? AND name=?;
*/});
exports.getModuleTotal = function (name, start, end, callback) {
mysql.query(SELECT_ONE_TOTAL_SQL, [start, end, name], callback);
};
var SELECT_ALL_TOTAL_SQL = 'SELECT date, sum(count) AS count \
FROM download_total \
WHERE date>=? AND date<=? \
GROUP BY date;';
var SELECT_ALL_TOTAL_SQL = multiline(function () {;/*
SELECT
date, sum(count) AS count
FROM
download_total
WHERE
date>=? AND date<=?
GROUP BY
date;
*/});
exports.getTotal = function (start, end, callback) {
mysql.query(SELECT_ALL_TOTAL_SQL, [start, end], callback);
};

View File

@@ -19,15 +19,29 @@ var utility = require('utility');
var eventproxy = require('eventproxy');
var config = require('../config');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, version, description, package, dist_tarball, dist_shasum, dist_size';
var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, \
version, description, package, dist_tarball, dist_shasum, dist_size';
var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, \
publish_time, author, name, version, package, dist_tarball, dist_shasum, dist_size, description) \
VALUES(now(), now(), ?, ?, ?, ?, ?, ?, ?, ?, ?) \
ON DUPLICATE KEY UPDATE gmt_modified=now(), publish_time=VALUES(publish_time), description=VALUES(description), \
author=VALUES(author), name=VALUES(name), version=VALUES(version), package=VALUES(package), \
dist_tarball=VALUES(dist_tarball), dist_shasum=VALUES(dist_shasum), dist_size=VALUES(dist_size);';
var INSERT_MODULE_SQL = multiline(function () {;/*
INSERT INTO
module(gmt_create, gmt_modified, publish_time, author, name, version,
package, dist_tarball, dist_shasum, dist_size, description)
VALUES
(now(), now(), ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
gmt_modified=now(),
publish_time=VALUES(publish_time),
description=VALUES(description),
author=VALUES(author),
name=VALUES(name),
version=VALUES(version),
package=VALUES(package),
dist_tarball=VALUES(dist_tarball),
dist_shasum=VALUES(dist_shasum),
dist_size=VALUES(dist_size);
*/});
exports.add = function (mod, callback) {
var keywords = mod.package.keywords;
@@ -82,7 +96,16 @@ exports.add = function (mod, callback) {
});
};
var GET_KEYWORD_SQL = 'SELECT keyword FROM module_keyword WHERE name=? ORDER BY keyword;';
var GET_KEYWORD_SQL = multiline(function () {;/*
SELECT
keyword
FROM
module_keyword
WHERE
name = ?
ORDER BY
keyword;
*/});
exports.getKeywords = function (name, callback) {
mysql.query(GET_KEYWORD_SQL, [name], function (err, rows) {
@@ -96,9 +119,14 @@ exports.getKeywords = function (name, callback) {
});
};
var ADD_KEYWORD_SQL = 'INSERT INTO module_keyword(gmt_create, keyword, name, description) \
VALUES(now(), ?, ?, ?) \
ON DUPLICATE KEY UPDATE description=VALUES(description);';
var ADD_KEYWORD_SQL = multiline(function () {;/*
INSERT INTO
module_keyword(gmt_create, keyword, name, description)
VALUES
(now(), ?, ?, ?)
ON DUPLICATE KEY UPDATE
description=VALUES(description);
*/});
exports.addKeywords = function (name, description, keywords, callback) {
var sql = '';
@@ -124,12 +152,26 @@ exports.addKeywords = function (name, description, keywords, callback) {
});
};
var UPDATE_DESC_SQL = 'UPDATE module SET description=? WHERE id=?;';
var UPDATE_DESC_SQL = multiline(function () {;/*
UPDATE
module
SET
description=?
WHERE
id=?;
*/});
exports.updateDescription = function (id, description, callback) {
mysql.query(UPDATE_DESC_SQL, [description, id], callback);
};
var UPDATE_PACKAGE_SQL = 'UPDATE module SET package=? WHERE id=?;';
var UPDATE_PACKAGE_SQL = multiline(function () {;/*
UPDATE
module
SET
package=?
WHERE
id=?;
*/});
exports.updateReadme = function (id, readme, callback) {
exports.getById(id, function (err, data) {
if (err) {
@@ -142,9 +184,19 @@ exports.updateReadme = function (id, readme, callback) {
});
};
var UPDATE_DIST_SQL = 'UPDATE module SET publish_time=?, version=?, package=?, \
dist_tarball=?, dist_shasum=?, dist_size=? WHERE id=?;';
var UPDATE_DIST_SQL = multiline(function () {;/*
UPDATE
module
SET
publish_time=?,
version=?,
package=?,
dist_tarball=?,
dist_shasum=?,
dist_size=?
WHERE
id=?;
*/});
exports.update = function (mod, callback) {
var pkg;
try {
@@ -182,8 +234,16 @@ function stringifyPackage(pkg) {
return encodeURIComponent(JSON.stringify(pkg));
}
var SELECT_MODULE_BY_ID_SQL = 'SELECT ' + MODULE_COLUMNS + ' FROM module WHERE id=?;';
var SELECT_MODULE_BY_ID_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
FROM
module
WHERE
id=?;
*/});
exports.getById = function (id, callback) {
id = Number(id);
mysql.queryOne(SELECT_MODULE_BY_ID_SQL, [id], function (err, row) {
@@ -200,8 +260,15 @@ exports.getById = function (id, callback) {
});
};
var SELECT_MODULE_SQL = 'SELECT ' + MODULE_COLUMNS + ' FROM module WHERE name=? AND version=?;';
var SELECT_MODULE_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
FROM
module
WHERE
name=? AND version=?;
*/});
exports.get = function (name, version, callback) {
mysql.queryOne(SELECT_MODULE_SQL, [name, version], function (err, row) {
if (err || !row) {
@@ -217,14 +284,26 @@ exports.get = function (name, version, callback) {
});
};
var INSERT_TAG_SQL = 'INSERT INTO tag(gmt_create, gmt_modified, \
name, tag, version, module_id) \
VALUES(now(), now(), ?, ?, ?, ?) \
ON DUPLICATE KEY UPDATE gmt_modified=now(), module_id=VALUES(module_id), \
name=VALUES(name), tag=VALUES(tag), version=VALUES(version);';
var SELECT_MODULE_ID_SQL = 'SELECT id FROM module WHERE name=? AND version=?;';
var SELECT_MODULE_ID_SQL = multiline(function () {;/*
SELECT
id
FROM
module
WHERE
name=? AND version=?;
*/});
var INSERT_TAG_SQL = multiline(function () {;/*
INSERT INTO
tag(gmt_create, gmt_modified, name, tag, version, module_id)
VALUES
(now(), now(), ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
gmt_modified=now(),
module_id=VALUES(module_id),
name=VALUES(name),
tag=VALUES(tag),
version=VALUES(version);
*/});
exports.addTag = function (name, tag, version, callback) {
mysql.queryOne(SELECT_MODULE_ID_SQL, [name, version], function (err, row) {
if (err) {
@@ -240,8 +319,14 @@ exports.addTag = function (name, tag, version, callback) {
});
};
var SELECT_TAG_SQL = 'SELECT tag, version, gmt_modified, module_id FROM tag WHERE name=? AND tag=?;';
var SELECT_TAG_SQL = multiline(function () {;/*
SELECT
tag, version, gmt_modified, module_id
FROM
tag
WHERE
name=? AND tag=?;
*/});
exports.getByTag = function (name, tag, callback) {
mysql.queryOne(SELECT_TAG_SQL, [name, tag], function (err, row) {
if (err || !row) {
@@ -251,26 +336,51 @@ exports.getByTag = function (name, tag, callback) {
});
};
var DELETE_TAGS_SQL = 'DELETE FROM tag WHERE name=?;';
var DELETE_TAGS_SQL = multiline(function () {;/*
DELETE FROM
tag
WHERE
name=?;
*/});
exports.removeTags = function (name, callback) {
mysql.query(DELETE_TAGS_SQL, [name], callback);
};
var DELETE_TAGS_BY_IDS_SQL = 'DELETE FROM tag WHERE id in (?)';
var DELETE_TAGS_BY_IDS_SQL = multiline(function () {;/*
DELETE FROM
tag
WHERE
id in (?);
*/});
exports.removeTagsByIds = function (ids, callback) {
mysql.query(DELETE_TAGS_BY_IDS_SQL, [ids], callback);
};
var SELECT_ALL_TAGS_SQL = 'SELECT id, tag, version, gmt_modified, module_id FROM tag WHERE name=?;';
var SELECT_ALL_TAGS_SQL = multiline(function () {;/*
SELECT
id, tag, version, gmt_modified, module_id
FROM
tag
WHERE
name=?;
*/});
exports.listTags = function (name, callback) {
mysql.query(SELECT_ALL_TAGS_SQL, [name], callback);
};
var SELECT_LATEST_MODULE_SQL = 'SELECT ' + MODULE_COLUMNS +
' FROM module WHERE name=? AND version <> "next" ORDER BY publish_time DESC LIMIT 1;';
var SELECT_LATEST_MODULE_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
FROM
module
WHERE
name=? AND version <> "next"
ORDER BY
publish_time DESC
LIMIT
1;
*/});
exports.getLatest = function (name, callback) {
exports.getByTag(name, 'latest', function (err, row) {
if (err || row) {
@@ -293,8 +403,17 @@ exports.getLatest = function (name, callback) {
});
};
var LIST_MODULE_SQL = 'SELECT ' + MODULE_COLUMNS + ' FROM module WHERE name=? ORDER BY id DESC;';
var LIST_MODULE_SQL = multiline(function () {;/*
SELECT
id, publish_time, gmt_create, gmt_modified, author, name,
version, description, package, dist_tarball, dist_shasum, dist_size
FROM
module
WHERE
name=?
ORDER BY
id DESC;
*/});
exports.listByName = function (name, callback) {
mysql.query(LIST_MODULE_SQL, [name], function (err, rows) {
if (err) {
@@ -314,10 +433,23 @@ exports.listByName = function (name, callback) {
});
};
var LIST_SINCE_SQLS = [
'SELECT module_id FROM tag WHERE tag="latest" AND gmt_modified>?',
'SELECT name, package FROM module WHERE id IN (?);'
];
var LIST_SINCE_SQLS = [];
LIST_SINCE_SQLS.push(multiline(function () {;/*
SELECT
module_id
FROM
tag
WHERE
tag="latest" AND gmt_modified>?;
*/}));
LIST_SINCE_SQLS.push(multiline(function () {;/*
SELECT
distinct(name)
FROM
module
WHERE
id IN (?);
*/}));
exports.listSince = function (start, callback) {
var ep = eventproxy.create();
ep.fail(callback);
@@ -335,31 +467,91 @@ exports.listSince = function (start, callback) {
});
};
var LIST_SHORT_SQL = 'SELECT distinct(name) FROM tag ORDER BY name';
var LIST_ALL_NAME_SQL = multiline(function () {;/*
SELECT
distinct(name)
FROM
module;
*/});
exports.listAllNames = function (callback) {
mysql.query(LIST_ALL_NAME_SQL, [], callback);
};
var LIST_SHORT_SQL = multiline(function () {;/*
SELECT
distinct(name)
FROM
tag
ORDER BY
name;
*/});
exports.listShort = function (callback) {
mysql.query(LIST_SHORT_SQL, callback);
};
var LIST_ALL_MODULE_NAMES_SQL = 'SELECT distinct(name) FROM module ORDER BY name';
var LIST_ALL_MODULE_NAMES_SQL = multiline(function () {;/*
SELECT
distinct(name)
FROM
module
ORDER BY
name;
*/});
exports.listAllModuleNames = function (callback) {
mysql.query(LIST_ALL_MODULE_NAMES_SQL, callback);
};
var DELETE_MODULE_BY_NAME_SQL = 'DELETE FROM module WHERE name=?;';
var DELETE_MODULE_BY_NAME_SQL = multiline(function () {;/*
DELETE FROM
module
WHERE
name=?;
*/});
exports.removeByName = function (name, callback) {
mysql.query(DELETE_MODULE_BY_NAME_SQL, [name], callback);
};
var DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL = 'DELETE FROM module WHERE name=? AND version IN(?);';
var DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL = multiline(function () {;/*
DELETE FROM
module
WHERE
name=? AND version in(?);
*/});
exports.removeByNameAndVersions = function (name, versions, callback) {
mysql.query(DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL, [name, versions], callback);
};
var LIST_BY_AUTH_SQLS = [
'SELECT distinct(name) AS name FROM module WHERE author = ? ORDER BY publish_time DESC LIMIT 100;',
'SELECT module_id FROM tag WHERE tag="latest" AND name IN (?)',
'SELECT name, description FROM module WHERE id IN (?) ORDER BY publish_time DESC'
];
var LIST_BY_AUTH_SQLS = [];
LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
SELECT
distinct(name) AS name
FROM
module
WHERE
author=?
ORDER BY
publish_time DESC
LIMIT
100;
*/}));
LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
SELECT
module_id
FROM
tag
WHERE
tag="latest" AND name IN (?);
*/}));
LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
SELECT
name, description
FROM
module
WHERE
id IN (?)
ORDER BY
publish_time DESC;
*/}));
exports.listByAuthor = function (author, callback) {
var ep = eventproxy.create();
ep.fail(callback);
@@ -386,10 +578,40 @@ exports.listByAuthor = function (author, callback) {
});
};
var SEARCH_MODULES_SQL = 'SELECT module_id FROM tag WHERE name LIKE ? AND tag="latest" ORDER BY name LIMIT ?;';
var SEARCH_MODULES_BY_KEYWORD_SQL = 'SELECT name, description FROM module_keyword WHERE keyword = ? ORDER BY id DESC LIMIT ?;';
var QUERY_MODULES_BY_ID_SQL = 'SELECT name, description FROM module WHERE id IN (?) ORDER BY name;';
var SEARCH_MODULES_SQL = multiline(function () {;/*
SELECT
module_id
FROM
tag
WHERE
name LIKE ? AND tag="latest"
ORDER BY
name
LIMIT
?;
*/});
var SEARCH_MODULES_BY_KEYWORD_SQL = multiline(function () {;/*
SELECT
name, description
FROM
module_keyword
WHERE
keyword=?
ORDER BY
id DESC
LIMIT
?;
*/});
var QUERY_MODULES_BY_ID_SQL = multiline(function () {;/*
SELECT
name, description
FROM
module
WHERE
id IN (?)
ORDER BY
name;
*/});
exports.search = function (word, options, callback) {
if (typeof options === 'function') {
callback = options;
@@ -452,3 +674,25 @@ exports.search = function (word, options, callback) {
};
thunkify(exports);
exports.updateMaintainers = function *(id, maintainers) {
var mod = yield exports.getById(id);
mod.package.maintainers = maintainers;
var pkg = stringifyPackage(mod.package);
return yield mysql.query(UPDATE_PACKAGE_SQL, [pkg, id]);
};
var GET_LAST_MODIFIED_MODULE_SQL = multiline(function () {;/*
SELECT
id, gmt_modified
FROM
module
WHERE
name=?
ORDER BY
gmt_modified DESC;
*/});
exports.getLastModified = function *(name) {
var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
return row && row.gmt_modified;
};

View File

@@ -16,16 +16,26 @@
var thunkify = require('thunkify-wrap');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var LIST_DEPS_SQL = 'SELECT deps FROM module_deps WHERE name=?;';
var LIST_DEPS_SQL = multiline(function () {;/*
SELECT
deps
FROM
module_deps
WHERE
name=?;
*/});
exports.list = function (name, callback) {
mysql.query(LIST_DEPS_SQL, [name], callback);
};
var INSERT_DEPS_SQL = 'INSERT INTO module_deps(gmt_create, name, deps) \
VALUES(now(), ?, ?);';
var INSERT_DEPS_SQL = multiline(function () {;/*
INSERT INTO
module_deps(gmt_create, name, deps)
VALUES
(now(), ?, ?);
*/});
exports.add = function (name, deps, callback) {
mysql.query(INSERT_DEPS_SQL, [name, deps], function (err, result) {
if (err && err.code === 'ER_DUP_ENTRY') {
@@ -35,8 +45,12 @@ exports.add = function (name, deps, callback) {
});
};
var DELETE_DEPS_SQL = 'DELETE FROM module_deps WHERE name=? AND deps=?;';
var DELETE_DEPS_SQL = multiline(function () {;/*
DELETE FROM
module_deps
WHERE
name=? AND deps=?;
*/});
exports.remove = function (name, deps, callback) {
mysql.query(DELETE_DEPS_SQL, [name, deps], callback);
};

View File

@@ -16,10 +16,14 @@
var thunkify = require('thunkify-wrap');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var INSERT_LOG_SQL = 'INSERT INTO module_log(gmt_create, gmt_modified, name, username, log) \
VALUES(now(), now(), ?, ?, "");';
var INSERT_LOG_SQL = multiline(function () {;/*
INSERT INTO
module_log(gmt_create, gmt_modified, name, username, log)
VALUES
(now(), now(), ?, ?, "");
*/});
exports.create = function (data, callback) {
mysql.query(INSERT_LOG_SQL, [data.name, data.username], function (err, result) {
if (err) {
@@ -29,7 +33,15 @@ exports.create = function (data, callback) {
});
};
var APPEND_SQL = 'UPDATE module_log SET log=CONCAT(log, ?), gmt_modified=now() WHERE id=?;';
var APPEND_SQL = multiline(function () {;/*
UPDATE
module_log
SET
log=CONCAT(log, ?),
gmt_modified=now()
WHERE
id=?;
*/});
exports.append = function (id, log, callback) {
log = '\n' + log;
mysql.query(APPEND_SQL, [log, id], function (err) {
@@ -37,7 +49,14 @@ exports.append = function (id, log, callback) {
});
};
var SELECT_SQL = 'SELECT * FROM module_log WHERE id=?;';
var SELECT_SQL = multiline(function () {;/*
SELECT
*
FROM
module_log
WHERE
id=?;
*/});
exports.get = function (id, callback) {
mysql.queryOne(SELECT_SQL, [id], callback);
};

73
proxy/module_star.js Normal file
View File

@@ -0,0 +1,73 @@
/**!
* cnpmjs.org - proxy/module_star.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var mysql = require('../common/mysql');
var multiline = require('multiline');
var ADD_SQL = multiline(function () {;/*
INSERT iNTO
module_star(name, user)
VALUES
(?, ?);
*/});
exports.add = function *add(name, user) {
try {
yield mysql.query(ADD_SQL, [name, user]);
} catch (err) {
if (err.code !== 'ER_DUP_ENTRY') {
throw err;
}
}
};
var REMOVE_SQL = multiline(function () {;/*
DELETE FROM
module_star
WHERE
name = ? AND user = ?;
*/});
exports.remove = function *(name, user) {
return yield mysql.query(REMOVE_SQL, [name, user]);
};
var LIST_USERS_SQL = multiline(function () {;/*
SELECT
user
FROM
module_star
WHERE
name = ?;
*/});
exports.listUsers = function *(name) {
var rows = yield mysql.query(LIST_USERS_SQL, [name]);
return rows.map(function (r) {
return r.user;
});
};
var LIST_USER_MODULES_SQL = multiline(function () {;/*
SELECT
name
FROM
module_star
WHERE
user = ?;
*/});
exports.listUserModules = function *(user) {
return (yield mysql.query(LIST_USER_MODULES_SQL, [user])).map(function (r) {
return r.name;
});
};

View File

@@ -14,56 +14,65 @@
* Module dependencies.
*/
var thunkify = require('thunkify-wrap');
var urllib = require('urllib');
var urllib = require('co-urllib');
var config = require('../config');
function request(url, options, callback) {
if (typeof options === 'function') {
callback = options;
options = null;
}
var USER_AGENT = 'cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
function *request(url, options) {
options = options || {};
options.dataType = options.dataType || 'json';
options.timeout = options.timeout || 120000;
url = config.sourceNpmRegistry + url;
urllib.request(url, options, function (err, data, res) {
if (err) {
var statusCode = res && res.statusCode || -1;
if (err.name === 'JSONResponseFormatError' && statusCode >= 500) {
err.name = 'NPMServerError';
err.message = 'Status ' + statusCode + ', ' + (data && data.toString() || 'empty body');
}
options.headers = {
'user-agent': USER_AGENT
};
var registry = options.registry || config.sourceNpmRegistry;
url = registry + url;
var r;
try {
r = yield *urllib.request(url, options);
} catch (err) {
var statusCode = err.status || -1;
var data = err.data || '[empty]';
if (err.name === 'JSONResponseFormatError' && statusCode >= 500) {
err.name = 'NPMServerError';
err.message = 'Status ' + statusCode + ', ' + data.toString();
}
callback(err, data, res);
});
throw err;
}
return r;
}
exports.get = function (name, callback) {
request('/' + name, function (err, data, res) {
if (err) {
return callback(err);
}
res = res || {};
if (res.statusCode === 404) {
data = null;
}
callback(null, data, res);
exports.getUser = function *(name) {
var url = '/-/user/org.couchdb.user:' + name;
var r = yield *request(url);
var data = r.data;
if (data && !data.name) {
data = null;
}
return data;
};
exports.get = function *(name) {
var r = yield *request('/' + name);
var data = r.data;
if (r.status === 404) {
data = null;
}
return data;
};
exports.getAllSince = function *(startkey) {
var r = yield *request('/-/all/since?stale=update_after&startkey=' + startkey, {
timeout: 300000
});
return r.data;
};
exports.getAllSince = function (startkey, callback) {
request('/-/all/since?stale=update_after&startkey=' + startkey, {
dataType: 'json',
timeout: 300000
}, callback);
exports.getShort = function *() {
var r = yield *request('/-/short', {
timeout: 300000,
registry: 'http://r.cnpmjs.org', // registry.npmjs.org/-/short is 404 now.
});
return r.data;
};
exports.getShort = function (callback) {
request('/-/short', {
dataType: 'json',
timeout: 300000
}, callback);
};
thunkify(exports);

View File

@@ -15,6 +15,9 @@
* Module dependencies.
*/
var co = require('co');
var gather = require('co-gather');
var defer = require('co-defer');
var thunkify = require('thunkify-wrap');
var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
var EventEmitter = require('events').EventEmitter;
@@ -22,8 +25,7 @@ var util = require('util');
var fs = require('fs');
var path = require('path');
var crypto = require('crypto');
var eventproxy = require('eventproxy');
var urllib = require('urllib');
var urllib = require('co-urllib');
var utility = require('utility');
var ms = require('ms');
var nfs = require('../common/nfs');
@@ -33,20 +35,20 @@ var Module = require('./module');
var ModuleDeps = require('./module_deps');
var Log = require('./module_log');
var config = require('../config');
var ModuleStar = require('./module_star');
var User = require('./user');
var USER_AGENT = 'sync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
function SyncModuleWorker(options) {
EventEmitter.call(this);
this._logId = options.logId;
this.startName = options.name;
if (!Array.isArray(options.name)) {
options.name = [options.name];
}
this.names = options.name;
// for (var i = 0; i < this.names.length; i++) {
// // ensure package name is lower case
// this.names[i] = this.names[i].toLowerCase();
// }
this.names = options.name || [];
this.startName = this.names[0];
this.username = options.username;
this.concurrency = options.concurrency || 1;
@@ -89,9 +91,24 @@ SyncModuleWorker.prototype.log = function (format, arg1, arg2) {
SyncModuleWorker.prototype.start = function () {
this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s',
this.username, this.names[0], this.concurrency, this.noDep, this._publish);
for (var i = 0; i < this.concurrency; i++) {
this.next(i);
}
var self = this;
co(function *() {
var arr = [];
for (var i = 0; i < self.concurrency; i++) {
arr.push(self.next(i));
}
yield arr;
})();
};
SyncModuleWorker.prototype.pushSuccess = function (name) {
this.successes.push(name);
this.emit('success', name);
};
SyncModuleWorker.prototype.pushFail = function (name) {
this.fails.push(name);
this.emit('fail', name);
};
SyncModuleWorker.prototype.add = function (name) {
@@ -100,322 +117,451 @@ SyncModuleWorker.prototype.add = function (name) {
}
this.nameMap[name] = true;
this.names.push(name);
this.emit('add', name);
this.log(' add dependencies: %s', name);
};
SyncModuleWorker.prototype.next = function (concurrencyId) {
SyncModuleWorker.prototype._doneOne = function *(concurrencyId, name, success) {
if (success) {
this.pushSuccess(name);
} else {
this.pushFail(name);
}
delete this.syncingNames[name];
var that = this;
// relase the stack: https://github.com/cnpm/cnpmjs.org/issues/328
defer.setImmediate(function *() {
that.log('[c#%s] setImmediate after, %s done, start next...', concurrencyId, name);
yield *that.next(concurrencyId);
});
};
SyncModuleWorker.prototype.next = function *(concurrencyId) {
var name = this.names.shift();
if (!name) {
return process.nextTick(this.finish.bind(this));
return setImmediate(this.finish.bind(this));
}
var that = this;
that.syncingNames[name] = true;
npm.get(name, function (err, pkg, response) {
var statusCode = response && response.statusCode || -1;
if (!err && !pkg) {
err = new Error('Module ' + name + ' not exists, http status ' + statusCode);
err.name = 'NpmModuleNotExsitsError';
}
if (err) {
if (statusCode === 404) {
that.successes.push(name);
} else {
that.fails.push(name);
}
that.log('[error] [%s] get package error: %s', name, err.stack);
delete that.syncingNames[name];
return that.next(concurrencyId);
var pkg;
// get from npm
try {
pkg = yield npm.get(name);
} catch (err) {
// if 404
if (!err.res || err.res.statusCode !== 404) {
var errMessage = err.name + ': ' + err.message;
that.log('[c#%s] [error] [%s] get package error: %s', concurrencyId, name, errMessage);
yield *that._doneOne(concurrencyId, name, false);
return;
}
}
that.log('[c#%d] [%s] start...', concurrencyId, name);
that._sync(name, pkg, function (err, versions) {
delete that.syncingNames[name];
if (err) {
that.fails.push(name);
that.log('[error] [%s] sync error: %s', name, err.stack);
return that.next(concurrencyId);
}
that.log('[%s] synced success, %d versions: %s',
name, versions.length, versions.join(', '));
that.successes.push(name);
that.emit('success', name);
that.next(concurrencyId);
});
});
if (!pkg) {
that.log('[c#%s] [error] [%s] get package error: package not exists', concurrencyId, name);
yield *that._doneOne(concurrencyId, name, true);
return;
}
that.log('[c#%d] [%s] start...', concurrencyId, name);
var versions;
try {
versions = yield that._sync(name, pkg);
} catch (err) {
that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
yield *that._doneOne(concurrencyId, name, false);
return;
}
that.log('[c#%d] [%s] synced success, %d versions: %s',
concurrencyId, name, versions.length, versions.join(', '));
yield *that._doneOne(concurrencyId, name, true);
};
SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
function *_listStarUsers(modName) {
var users = yield ModuleStar.listUsers(modName);
var userMap = {};
users.forEach(function (user) {
userMap[user] = true;
});
return userMap;
}
function *_addStar(modName, username) {
yield ModuleStar.add(modName, username);
}
function *_saveNpmUser(username) {
var user = yield *npm.getUser(username);
if (!user) {
return;
}
yield User.saveNpmUser(user);
}
SyncModuleWorker.prototype._sync = function *(name, pkg) {
var username = this.username;
var that = this;
var ep = eventproxy.create();
ep.fail(callback);
var hasModules = false;
Module.listByName(name, ep.done(function (rows) {
hasModules = rows.length > 0;
var map = {};
for (var i = 0; i < rows.length; i++) {
var r = rows[i];
if (!r.package || !r.package.dist) {
// package json parse error
continue;
}
var result = yield [
Module.listByName(name),
Module.listTags(name),
_listStarUsers(name)
];
var moduleRows = result[0];
var tagRows = result[1];
var existsStarUsers = result[2];
if (r.package && r.package._publish_on_cnpm) {
// publish on cnpm, dont sync this version package
that.log(' [%s] publish on local cnpm, don\'t sync', name);
ep.unbind();
callback(null, []);
return;
}
if (r.version === 'next') {
continue;
}
if (!map.latest) {
map.latest = r;
}
map[r.version] = r;
hasModules = moduleRows.length > 0;
var map = {};
for (var i = 0; i < moduleRows.length; i++) {
var r = moduleRows[i];
if (!r.package || !r.package.dist) {
// package json parse error
continue;
}
ep.emit('existsMap', map);
}));
Module.listTags(name, ep.done(function (rows) {
var tags = {};
for (var i = 0; i < rows.length; i++) {
var r = rows[i];
if (!r.module_id) {
// no module_id, need to sync tags
continue;
}
tags[r.tag] = r.version;
if (r.package && r.package._publish_on_cnpm) {
// publish on cnpm, dont sync this version package
that.log(' [%s] publish on local cnpm, don\'t sync', name);
return [];
}
ep.emit('existsTags', tags);
}));
if (r.version === 'next') {
continue;
}
if (!map.latest) {
map.latest = r;
}
map[r.version] = r;
}
var tags = {};
for (var i = 0; i < tagRows.length; i++) {
var r = tagRows[i];
if (!r.module_id) {
// no module_id, need to sync tags
continue;
}
tags[r.tag] = r.version;
}
var missingVersions = [];
var missingTags = [];
var missingDescriptions = [];
var missingReadmes = [];
var missingStarUsers = [];
var npmUsernames = {};
ep.all('existsMap', 'existsTags', function (map, tags) {
var times = pkg.time || {};
pkg.versions = pkg.versions || {};
var versionNames = Object.keys(times);
if (versionNames.length === 0) {
versionNames = Object.keys(pkg.versions);
// find out all user names
for (var v in pkg.versions) {
var p = pkg.versions[v];
var maintainers = p.maintainers || [];
if (maintainers && !Array.isArray(maintainers)) {
// http://r.cnpmjs.org/jasmine-node
// TODO: "maintainers": "Martin Häger <martin.haeger@gmail.com>",
maintainers = [maintainers];
}
if (versionNames.length === 0) {
that.log(' [%s] no times and no versions, hasModules: %s', name, hasModules);
if (!hasModules) {
// save a next module
var maintainer = pkg.maintainers && pkg.maintainers[0];
if (maintainer && maintainer.name) {
maintainer = maintainer.name;
}
if (!maintainer) {
maintainer = '-';
}
var nextMod = {
maintainers.forEach(function (m) {
if (m.name) {
npmUsernames[m.name.toLowerCase()] = 1;
}
});
}
// get the missing star users
var starUsers = pkg.users || {};
for (var k in starUsers) {
if (!existsStarUsers[k]) {
missingStarUsers.push(k);
}
npmUsernames[k.toLowerCase()] = 1;
}
that.log(' [%s] found %d missing star users', name, missingStarUsers.length);
var times = pkg.time || {};
pkg.versions = pkg.versions || {};
var versionNames = Object.keys(times);
if (versionNames.length === 0) {
versionNames = Object.keys(pkg.versions);
}
if (versionNames.length === 0) {
that.log(' [%s] no times and no versions, hasModules: %s', name, hasModules);
if (!hasModules) {
// save a next module
var maintainer = pkg.maintainers && pkg.maintainers[0];
if (maintainer && maintainer.name) {
maintainer = maintainer.name;
}
if (!maintainer) {
maintainer = '-';
}
var nextMod = {
name: name,
version: 'next',
author: maintainer,
package: {
name: name,
version: 'next',
author: maintainer,
package: {
name: name,
version: 'next',
description: pkg.description || '',
readme: pkg.readme || '',
maintainers: pkg.maintainers || {
name: maintainer
},
description: pkg.description || '',
readme: pkg.readme || '',
maintainers: pkg.maintainers || {
name: maintainer
},
};
Module.add(nextMod, function (err, result) {
that.log(' [%s] save next module, %j, error: %s', name, result, err);
});
},
};
try {
var result = yield Module.add(nextMod);
that.log(' [%s] save next module, %j', name, result);
} catch (err) {
that.log(' [%s] save next module error %s', err.message);
}
}
var versions = [];
for (var i = 0; i < versionNames.length; i++) {
var v = versionNames[i];
var exists = map[v] || {};
var version = pkg.versions[v];
if (!version || !version.dist) {
}
var versions = [];
for (var i = 0; i < versionNames.length; i++) {
var v = versionNames[i];
var exists = map[v] || {};
var version = pkg.versions[v];
if (!version || !version.dist || !version.dist.tarball) {
continue;
}
//patch for readme
if (!version.readme) {
version.readme = pkg.readme;
}
var publish_time = times[v];
version.publish_time = publish_time ? Date.parse(publish_time) : null;
if (!version.maintainers || !version.maintainers[0]) {
version.maintainers = pkg.maintainers;
}
var sourceAuthor = version.maintainers && version.maintainers[0] &&
version.maintainers[0].name || exists.author;
if (exists.package && exists.package.dist.shasum === version.dist.shasum &&
exists.author === sourceAuthor) {
// * author make sure equal
// * shasum make sure equal
if ((version.publish_time === exists.publish_time) ||
(!version.publish_time && exists.publish_time)) {
// debug(' [%s] %s publish_time equal: %s, %s',
// name, version.version, version.publish_time, exists.publish_time);
// * publish_time make sure equal
if (exists.description === null && version.description) {
// * make sure description exists
missingDescriptions.push({
id: exists.id,
description: version.description
});
}
if (!exists.package.readme && version.readme) {
// * make sure readme exists
missingReadmes.push({
id: exists.id,
readme: version.readme
});
}
continue;
}
//patch for readme
if (!version.readme) {
version.readme = pkg.readme;
}
var publish_time = times[v];
version.publish_time = publish_time ? Date.parse(publish_time) : null;
if (!version.maintainers || !version.maintainers[0]) {
version.maintainers = pkg.maintainers;
}
var sourceAuthor = version.maintainers && version.maintainers[0] && version.maintainers[0].name || exists.author;
if (exists.package && exists.package.dist.shasum === version.dist.shasum && exists.author === sourceAuthor) {
// * author make sure equal
// * shasum make sure equal
if ((version.publish_time === exists.publish_time) || (!version.publish_time && exists.publish_time)) {
debug(' [%s] %s publish_time equal: %s, %s',
name, version.version, version.publish_time, exists.publish_time);
// * publish_time make sure equal
if (exists.description === null && version.description) {
// * make sure description exists
missingDescriptions.push({
id: exists.id,
description: version.description
});
}
if (!exists.package.readme && version.readme) {
// * make sure readme exists
missingReadmes.push({
id: exists.id,
readme: version.readme
});
}
continue;
}
}
versions.push(version);
}
versions.push(version);
}
var sourceTags = pkg['dist-tags'] || {};
for (var t in sourceTags) {
var sourceTagVersion = sourceTags[t];
if (sourceTagVersion && tags[t] !== sourceTagVersion) {
missingTags.push([t, sourceTagVersion]);
}
}
if (versions.length === 0) {
that.log(' [%s] all versions are exists', name);
return ep.emit('syncDone');
var sourceTags = pkg['dist-tags'] || {};
for (var t in sourceTags) {
var sourceTagVersion = sourceTags[t];
if (sourceTagVersion && tags[t] !== sourceTagVersion) {
missingTags.push([t, sourceTagVersion]);
}
}
if (versions.length === 0) {
that.log(' [%s] all versions are exists', name);
} else {
versions.sort(function (a, b) {
return a.publish_time - b.publish_time;
});
missingVersions = versions;
that.log(' [%s] %d versions', name, versions.length);
ep.emit('syncModule', missingVersions.shift());
});
that.log(' [%s] %d versions need to sync', name, versions.length);
}
missingVersions = versions;
var versionNames = [];
var syncIndex = 0;
ep.on('syncModule', function (syncModule) {
// sync missing versions
while (missingVersions.length) {
var index = syncIndex++;
that._syncOneVersion(index, syncModule, function (err, result) {
if (err) {
that.log(' [%s:%d] error, version: %s, %s: %s',
syncModule.name, index, syncModule.version, err.name, err.message);
} else {
versionNames.push(syncModule.version);
}
var syncModule = missingVersions.shift();
if (!syncModule.dist.tarball) {
continue;
}
try {
var result = yield that._syncOneVersion(index, syncModule);
versionNames.push(syncModule.version);
} catch (err) {
that.log(' [%s:%d] error, version: %s, %s: %s',
syncModule.name, index, syncModule.version, err.name, err.message);
}
}
var nextVersion = missingVersions.shift();
if (!nextVersion) {
return ep.emit('syncDone', result);
}
ep.emit('syncModule', nextVersion);
});
});
ep.on('syncDone', function () {
// sync missing descriptions
function *syncDes() {
if (missingDescriptions.length === 0) {
return ep.emit('descriptionDone');
return;
}
that.log(' [%s] saving %d descriptions', name, missingDescriptions.length);
missingDescriptions.forEach(function (item) {
Module.updateDescription(item.id, item.description, function (err, result) {
if (err) {
that.log(' save error, id %s, description: %s, error: %s', item.id, item.description, err);
} else {
that.log(' saved, id: %s, description length: %d', item.id, item.description.length);
}
ep.emitLater('saveDescription');
});
});
var res = yield gather(missingDescriptions.map(function (item) {
return Module.updateDescription(item.id, item.description);
}));
ep.after('saveDescription', missingDescriptions.length, function () {
ep.emit('descriptionDone');
});
});
ep.on('syncDone', function () {
if (missingTags.length === 0) {
return ep.emit('tagDone');
for (var i = 0; i < res.length; i++) {
var item = missingDescriptions[i];
var r = res[i];
if (r.error) {
that.log(' save error, id: %s, description: %s, error: %s',
item.id, item.description, r.error.message);
} else {
that.log(' saved, id: %s, description length: %d',
item.id, item.description.length);
}
}
}
// sync missing tags
function *syncTag() {
if (missingTags.length === 0) {
return;
}
that.log(' [%s] adding %d tags', name, missingTags.length);
// sync tags
missingTags.forEach(function (item) {
Module.addTag(name, item[0], item[1], ep.done(function (result) {
that.log(' added tag %s:%s, module_id: %s', item[0], item[1], result && result.module_id);
ep.emitLater('addTag');
}));
});
ep.after('addTag', missingTags.length, function () {
ep.emit('tagDone');
});
});
var res = yield gather(missingTags.map(function (item) {
return Module.addTag(name, item[0], item[1]);
}));
ep.once('syncDone', function () {
for (var i = 0; i < res.length; i++) {
var item = missingTags[i];
var r = res[i];
if (r.error) {
that.log(' add tag %s:%s error, error: %s',
item.id, item.description, r.error.message);
} else {
that.log(' added tag %s:%s, module_id: %s',
item[0], item[1], r.value && r.value.module_id);
}
}
}
// sycn missing readme
function *syncReadme() {
if (missingReadmes.length === 0) {
return ep.emit('readmeDone');
return;
}
that.log(' [%s] saving %d readmes', name, missingReadmes.length);
var res = yield gather(missingReadmes.map(function (item) {
return Module.updateReadme(item.id, item.readme);
}));
for (var i = 0; i < res.length; i++) {
var item = missingReadmes[i];
var r = res[i];
if (r.error) {
that.log(' save error, id: %s, error: %s', item.id, r.error.message);
} else {
that.log(' saved, id: %s', item.id);
}
}
}
function *syncMissingUsers() {
var missingUsers = [];
var names = Object.keys(npmUsernames);
if (names.length === 0) {
return;
}
var rows = yield *User.listByNames(names);
var map = {};
rows.forEach(function (r) {
map[r.name] = r;
});
names.forEach(function (username) {
var r = map[username];
if (!r || !r.json) {
missingUsers.push(username);
}
});
if (missingUsers.length === 0) {
that.log(' [%s] all %d npm users exists', name, names.length);
return;
}
that.log(' [%s] saving %d readmes', name, missingReadmes.length);
missingReadmes.forEach(function (item) {
Module.updateReadme(item.id, item.readme, function (err, result) {
if (err) {
that.log(' save error, id: %s, error: %s', item.id, err);
} else {
that.log(' saved, id: %s', item.id);
}
ep.emitLater('saveReadme');
});
});
that.log(' [%s] saving %d/%d missing npm users: %j',
name, missingUsers.length, names.length, missingUsers);
var res = yield gather(missingUsers.map(function (username) {
return _saveNpmUser(username);
}));
ep.after('saveReadme', missingReadmes.length, function () {
ep.emit('readmeDone');
});
});
for (var i = 0; i < res.length; i++) {
var r = res[i];
if (r.error) {
that.log(' save npm user error, %s', r.error.message);
}
}
}
ep.all('tagDone', 'descriptionDone', 'readmeDone', function () {
// TODO: set latest version
callback(null, versionNames);
});
// sync missing star users
function *syncMissingStarUsers() {
if (missingStarUsers.length === 0) {
return;
}
that.log(' [%s] saving %d star users', name, missingStarUsers.length);
var res = yield gather(missingStarUsers.map(function (username) {
return _addStar(name, username);
}));
for (var i = 0; i < res.length; i++) {
var r = res[i];
if (r.error) {
that.log(' add star user error, %s', r.error.message);
}
}
}
yield [syncDes(), syncTag(), syncReadme(), syncMissingStarUsers(), syncMissingUsers()];
return versionNames;
};
SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePackage, callback) {
SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
var that = this;
var username = this.username;
var downurl = sourcePackage.dist.tarball;
var filename = path.basename(downurl);
var filepath = common.getTarballFilepath(filename);
var ws = fs.createWriteStream(filepath);
var options = {
writeStream: ws,
followRedirect: true,
timeout: 600000, // 10 minutes download
headers: {
'user-agent': USER_AGENT
}
};
var ep = eventproxy.create();
ep.fail(function (err) {
// remove tmp file whatever
fs.unlink(filepath, utility.noop);
callback(err);
});
var dependencies = Object.keys(sourcePackage.dependencies || {});
var devDependencies = Object.keys(sourcePackage.devDependencies || {});
that.log(' [%s:%d] syncing, version: %s, dist: %j, no deps: %s, publish on cnpm: %s, dependencies: %d, devDependencies: %d',
that.log(' [%s:%d] syncing, version: %s, dist: %j, no deps: %s, ' +
'publish on cnpm: %s, dependencies: %d, devDependencies: %d',
sourcePackage.name, versionIndex, sourcePackage.version,
sourcePackage.dist, that.noDep, that._publish,
dependencies.length, devDependencies.length);
@@ -438,60 +584,77 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
}
}
// add deps relations
dependencies.forEach(function (depName) {
ModuleDeps.add(depName, sourcePackage.name, utility.noop);
});
// add module dependence
try {
yield dependencies.map(function (depName) {
return ModuleDeps.add(depName, sourcePackage.name);
});
} catch (err) {
// ignore
}
var shasum = crypto.createHash('sha1');
var dataSize = 0;
urllib.request(downurl, options, ep.done(function (_, response) {
var statusCode = response && response.statusCode || -1;
if (statusCode === 404) {
// just copy source dist
shasum = sourcePackage.dist.shasum;
return ep.emit('uploadResult', {
url: downurl
});
}
try {
// get tarball
var r = yield *urllib.request(downurl, options);
var statusCode = r.status || -1;
// https://github.com/cnpm/cnpmjs.org/issues/325
// if (statusCode === 404) {
// shasum = sourcePackage.dist.shasum;
// return yield *afterUpload({
// url: downurl
// });
// }
if (statusCode !== 200) {
var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
err.name = 'DownloadTarballError';
err.data = sourcePackage;
return ep.emit('error', err);
throw err;
}
// read and check
var rs = fs.createReadStream(filepath);
rs.once('error', function (err) {
ep.emit('error', err);
});
rs.on('data', function (data) {
shasum.update(data);
dataSize += data.length;
});
rs.on('end', function () {
shasum = shasum.digest('hex');
if (shasum !== sourcePackage.dist.shasum) {
var err = new Error('Download ' + downurl + ' shasum:' + shasum + ' not match ' + sourcePackage.dist.shasum);
err.name = 'DownloadTarballShasumError';
err.data = sourcePackage;
return ep.emit('error', err);
}
var end = thunkify.event(rs);
yield end(); // after end event emit
var options = {
key: common.getCDNKey(sourcePackage.name, filename),
size: dataSize,
shasum: shasum
};
nfs.upload(filepath, options, ep.done('uploadResult'));
});
}));
if (dataSize === 0) {
var err = new Error('Download ' + downurl + ' file size is zero');
err.name = 'DownloadTarballSizeZeroError';
err.data = sourcePackage;
throw err;
}
ep.on('uploadResult', function (result) {
// check shasum
shasum = shasum.digest('hex');
if (shasum !== sourcePackage.dist.shasum) {
var err = new Error('Download ' + downurl + ' shasum:' + shasum +
' not match ' + sourcePackage.dist.shasum);
err.name = 'DownloadTarballShasumError';
err.data = sourcePackage;
throw err;
}
options = {
key: common.getCDNKey(sourcePackage.name, filename),
size: dataSize,
shasum: shasum
};
// upload to NFS
var result = yield nfs.upload(filepath, options);
return yield *afterUpload(result);
} finally {
// remove tmp file whatever
fs.unlink(filepath, utility.noop);
}
function *afterUpload(result) {
//make sure sync module have the correct author info
//only if can not get maintainers, use the username
var author = username;
@@ -507,6 +670,8 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
publish_time: sourcePackage.publish_time,
};
// delete _publish_on_cnpm, because other cnpm maybe sync from current cnpm
delete mod.package._publish_on_cnpm;
if (that._publish) {
// sync as publish
mod.package._publish_on_cnpm = true;
@@ -526,54 +691,41 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
}
mod.package.dist = dist;
Module.add(mod, ep.done(function (result) {
that.log(' [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j, publish on cnpm: %s',
sourcePackage.name, versionIndex,
result.id,
author, mod.version, dataSize,
new Date(mod.publish_time),
that._publish);
callback(null, result);
}));
});
};
var r = yield Module.add(mod);
SyncModuleWorker.sync = function (name, username, options, callback) {
if (typeof options === 'function') {
callback = options;
options = null;
that.log(' [%s:%s] done, insertId: %s, author: %s, version: %s, '
+ 'size: %d, publish_time: %j, publish on cnpm: %s',
sourcePackage.name, versionIndex,
r.id,
author, mod.version, dataSize,
new Date(mod.publish_time),
that._publish);
return r;
}
options = options || {};
npm.get(name, function (err, pkg, response) {
if (err) {
return callback(err);
}
if (!pkg || !pkg._rev) {
return callback(null, {
ok: false,
statusCode: response.statusCode,
pkg: pkg
});
}
Log.create({name: name, username: username}, function (err, result) {
if (err) {
return callback(err);
}
var worker = new SyncModuleWorker({
logId: result.id,
name: name,
username: username,
noDep: options.noDep,
publish: options.publish,
});
worker.start();
callback(null, {
ok: true,
logId: result.id,
pkg: pkg
});
});
});
};
SyncModuleWorker.sync = thunkify(SyncModuleWorker.sync);
SyncModuleWorker.sync = function *(name, username, options) {
options = options || {};
var pkg = yield npm.get(name);
if (!pkg || !pkg._rev) {
return {
ok: false,
pkg: pkg
};
}
var result = yield Log.create({name: name, username: username});
var worker = new SyncModuleWorker({
logId: result.id,
name: name,
username: username,
noDep: options.noDep,
publish: options.publish,
});
worker.start();
return {
ok: true,
logId: result.id,
pkg: pkg
};
};

View File

@@ -18,13 +18,22 @@ var thunkify = require('thunkify-wrap');
var eventproxy = require('eventproxy');
var config = require('../config');
var mysql = require('../common/mysql');
var multiline = require('multiline');
var DB_SIZE_SQL = 'SELECT TABLE_NAME AS name, data_length, index_length \
FROM information_schema.tables \
WHERE TABLE_SCHEMA = ? \
GROUP BY TABLE_NAME \
ORDER BY data_length DESC \
LIMIT 0 , 200';
var DB_SIZE_SQL = multiline(function () {;/*
SELECT
TABLE_NAME AS name, data_length, index_length
FROM
information_schema.tables
WHERE
TABLE_SCHEMA = ?
GROUP BY
TABLE_NAME
ORDER BY
data_length DESC
LIMIT
0, 200;
*/});
var TOTAL_MODULE_SQL = 'SELECT count(distinct(name)) AS count FROM module;';
var TOTAL_VERSION_SQL = 'SELECT count(name) AS count FROM module;';
@@ -84,7 +93,14 @@ exports.get = function (callback) {
});
};
var PLUS_DELETE_MODULE_SQL = 'UPDATE total SET module_delete=module_delete+1 WHERE name="total";';
var PLUS_DELETE_MODULE_SQL = multiline(function () {;/*
UPDATE
total
SET
module_delete=module_delete+1
WHERE
name="total";
*/});
exports.plusDeleteModule = function (callback) {
mysql.query(PLUS_DELETE_MODULE_SQL, callback);
};
@@ -93,24 +109,55 @@ exports.getTotalInfo = function (callback) {
mysql.queryOne(TOTAL_INFO_SQL, callback);
};
var SET_LAST_SYNC_TIME_SQL = 'UPDATE total SET last_sync_time=? WHERE name="total";';
var SET_LAST_SYNC_TIME_SQL = multiline(function () {;/*
UPDATE
total
SET
last_sync_time=?
WHERE
name="total";
*/});
exports.setLastSyncTime = function (time, callback) {
mysql.query(SET_LAST_SYNC_TIME_SQL, Number(time), callback);
};
var SET_LAST_EXIST_SYNC_TIME_SQL = 'UPDATE total SET last_exist_sync_time=? WHERE name="total";';
var SET_LAST_EXIST_SYNC_TIME_SQL = multiline(function () {;/*
UPDATE
total
SET
last_exist_sync_time=?
WHERE
name="total";
*/});
exports.setLastExistSyncTime = function (time, callback) {
mysql.query(SET_LAST_EXIST_SYNC_TIME_SQL, Number(time), callback);
};
var UPDATE_SYNC_STATUS_SQL = 'UPDATE total SET sync_status = ? WHERE name="total";';
var UPDATE_SYNC_STATUS_SQL = multiline(function () {;/*
UPDATE
total
SET
sync_status=?
WHERE
name="total";
*/});
exports.updateSyncStatus = function (status, callback) {
mysql.query(UPDATE_SYNC_STATUS_SQL, [status], callback);
};
var UPDATE_SYNC_NUM_SQL = 'UPDATE total SET sync_status = ?, need_sync_num = ?, \
success_sync_num = ?, fail_sync_num = ?, left_sync_num = ?, last_sync_module=? \
WHERE name="total";';
var UPDATE_SYNC_NUM_SQL = multiline(function () {;/*
UPDATE
total
SET
sync_status = ?,
need_sync_num = ?,
success_sync_num = ?,
fail_sync_num = ?,
left_sync_num = ?,
last_sync_module = ?
WHERE
name="total";
*/});
exports.updateSyncNum = function (params, callback) {
var query = [
params.syncStatus, params.need || 0,

View File

@@ -18,32 +18,39 @@ var thunkify = require('thunkify-wrap');
var utility = require('utility');
var config = require('../config');
var mysql = require('../common/mysql');
var crypto = require('crypto');
var COLUMNS = 'id, rev, name, email, salt, password_sha, ip, roles, gmt_create, gmt_modified';
var SELECT_USER_SQL = 'SELECT ' + COLUMNS + ' FROM user WHERE name=?;';
function sha1(s) {
return crypto.createHash("sha1").update(s).digest("hex");
}
function passwordSha(password, salt) {
return sha1(password + salt);
}
var multiline = require('multiline');
var SELECT_USER_SQL = multiline(function () {;/*
SELECT
id, rev, name, email, salt, password_sha, ip,
roles, json, npm_user, gmt_create, gmt_modified
FROM
user
WHERE
name=?;
*/});
exports.get = function (name, callback) {
mysql.queryOne(SELECT_USER_SQL, [name], function (err, row) {
if (row) {
try {
row.roles = JSON.parse(row.roles);
row.roles = row.roles ? JSON.parse(row.roles) : [];
} catch (e) {
row.roles = [];
}
try {
row.json = row.json ? JSON.parse(row.json) : null;
} catch (e) {
row.json = null;
}
}
callback(err, row);
});
};
function passwordSha(password, salt) {
return utility.sha1(password + salt);
}
exports.auth = function (name, password, callback) {
exports.get(name, function (err, row) {
if (err || !row) {
@@ -58,9 +65,14 @@ exports.auth = function (name, password, callback) {
});
};
var INSERT_USER_SQL = 'INSERT INTO user(rev, name, email, salt, password_sha, ip, roles, gmt_create, gmt_modified) \
VALUES(?, ?, ?, ?, ?, ?, ?, now(), now())';
var INSERT_USER_SQL = multiline(function () {;/*
INSERT INTO
user(rev, name, email, salt, password_sha,
ip, roles, gmt_create, gmt_modified)
VALUES
(?, ?, ?, ?, ?, ?, ?, now(), now());
*/});
exports.add = function (user, callback) {
var roles = user.roles || [];
try {
@@ -75,9 +87,20 @@ exports.add = function (user, callback) {
});
};
var UPDATE_USER_SQL = 'UPDATE user SET rev=?, email=?, salt=?, password_sha=?, ip=?, roles=?, gmt_modified=now() \
WHERE name=? AND rev=?;';
var UPDATE_USER_SQL = multiline(function () {;/*
UPDATE
user
SET
rev=?,
email=?,
salt=?,
password_sha=?,
ip=?,
roles=?,
gmt_modified=now()
WHERE
name=? AND rev=?;
*/});
exports.update = function (user, callback) {
var rev = user.rev || user._rev;
var revNo = Number(rev.split('-', 1));
@@ -98,12 +121,41 @@ exports.update = function (user, callback) {
var values = [newRev, user.email, user.salt, user.password_sha, user.ip, roles, user.name, rev];
mysql.query(UPDATE_USER_SQL, values, function (err, data) {
if (err || !data.affectedRows) {
if (err) {
return callback(err);
}
callback(null, {rev: newRev});
callback(null, {rev: newRev, result: data});
});
};
thunkify(exports);
exports.passwordSha = passwordSha;
exports.saveNpmUser = function *(user) {
var sql = 'SELECT id, json FROM user WHERE name=?;';
var row = yield mysql.queryOne(sql, [user.name]);
if (!row) {
sql = 'INSERT INTO user(npm_user, json, rev, name, email, salt, password_sha, ip, gmt_create, gmt_modified) \
VALUES(1, ?, ?, ?, ?, "0", "0", "0", now(), now());';
yield mysql.query(sql, [JSON.stringify(user), user._rev, user.name, user.email]);
} else {
sql = 'UPDATE user SET json=?, rev=? WHERE id=?;';
yield mysql.query(sql, [JSON.stringify(user), user._rev, row.id]);
}
};
var LIST_BY_NAMES_SQL = multiline(function () {;/*
SELECT
id, name, email, json
FROM
user
WHERE
name in (?);
*/});
exports.listByNames = function *(names) {
if (names.length === 0) {
return [];
}
return yield mysql.query(LIST_BY_NAMES_SQL, [names]);
};

View File

@@ -16,6 +16,7 @@
*/
var middlewares = require('koa-middlewares');
var limit = require('../middleware/limit');
var login = require('../middleware/login');
var publishable = require('../middleware/publishable');
var syncByInstall = require('../middleware/sync_by_install');
@@ -44,7 +45,10 @@ function routes(app) {
app.put('/:name/sync', sync.sync);
app.get('/:name/sync/log/:id', sync.getSyncLog);
app.get('/:name/download/:filename', mod.download);
app.put('/:name/:tag', login, mod.updateTag);
// need limit by ip
app.get('/:name/download/:filename', limit, mod.download);
// put tarball
// https://registry.npmjs.org/cnpmjs.org/-/cnpmjs.org-0.0.0.tgz/-rev/1-c85bc65e8d2470cc4d82b8f40da65b8e
@@ -56,7 +60,7 @@ function routes(app) {
app.put('/:name/:version/-tag/latest', login, publishable, mod.updateLatest);
// update module, unpublish will PUT this
app.put('/:name/-rev/:rev', login, publishable, mod.removeWithVersions);
app.put('/:name/-rev/:rev', login, publishable, mod.updateOrRemove);
app.delete('/:name/-rev/:rev', login, publishable, mod.removeAll);
// try to create a new user

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - routes/web.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -19,6 +19,7 @@ var pkg = require('../controllers/web/package');
var user = require('../controllers/web/user');
var sync = require('../controllers/sync');
var total = require('../controllers/total');
var dist = require('../controllers/web/dist');
function routes(app) {
app.get('/total', total.show);
@@ -34,6 +35,8 @@ function routes(app) {
app.get('/sync', pkg.displaySync);
app.get('/_list/search/search', pkg.rangeSearch);
app.get(/^\/dist(\/.+)?/, dist.redirect);
}
module.exports = routes;

View File

@@ -18,30 +18,30 @@
var koa = require('koa');
var app = module.exports = koa();
var http = require('http');
var forward = require('forward');
var path = require('path');
var microtime = require('microtime');
var middlewares = require('koa-middlewares');
var routes = require('../routes/registry');
var logger = require('../common/logger');
var config = require('../config');
var session = require('../common/session');
var auth = require('../middleware/auth');
var staticCache = require('../middleware/static');
var notFound = require('../middleware/registry_not_found');
var rootdir = path.dirname(__dirname);
app.use(middlewares.rt({headerName: 'X-ReadTime'}));
app.use(middlewares.rewrite('/favicon.ico', '/public/favicon.ico'));
app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
app.use(staticCache);
app.keys = ['todokey', config.sessionSecret];
app.outputErrors = true;
app.proxy = true;
app.use(session);
app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
app.use(auth());
app.use(notFound);
app.use(middlewares.gzip());
app.use(middlewares.fresh());
app.use(middlewares.compress({threshold: 150}));
app.use(middlewares.conditional());
app.use(middlewares.etag());
/**

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - servers/web.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -18,12 +18,15 @@
var path = require('path');
var http = require('http');
var fs = require('fs');
var microtime = require('microtime');
var koa = require('koa');
var middlewares = require('koa-middlewares');
var markdown = require('koa-markdown');
var session = require('../common/session');
var opensearch = require('../middleware/opensearch');
var notFound = require('../middleware/web_not_found');
var staticCache = require('../middleware/static');
var auth = require('../middleware/auth');
var routes = require('../routes/web');
var logger = require('../common/logger');
var config = require('../config');
@@ -32,21 +35,21 @@ var app = koa();
var rootdir = path.dirname(__dirname);
app.use(middlewares.rt({headerName: 'X-ReadTime'}));
app.use(middlewares.staticCache(path.join(__dirname, '..', 'public'), {
buffer: !config.debug,
maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
dir: path.join(rootdir, 'public')
}));
app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
app.use(staticCache);
app.use(opensearch);
app.keys = ['todokey', config.sessionSecret];
app.outputErrors = true;
app.proxy = true;
app.use(session);
app.use(middlewares.bodyParser());
app.use(auth());
app.use(notFound);
app.use(middlewares.gzip());
app.use(middlewares.fresh());
app.use(middlewares.compress({threshold: 150}));
app.use(middlewares.conditional());
app.use(middlewares.etag());
var viewDir = path.join(rootdir, 'view', 'web');
@@ -59,13 +62,23 @@ var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
.replace('{{logoURL}}', config.logoURL);
fs.writeFileSync(layoutFile, layout);
// custom web readme home page support
var readmeFile = path.join(docDir, '_readme.md');
var readmeContent;
if (config.customReadmeFile) {
readmeContent = fs.readFileSync(config.customReadmeFile, 'utf8');
} else {
readmeContent = fs.readFileSync(path.join(docDir, 'readme.md'), 'utf8');
}
fs.writeFileSync(readmeFile, readmeContent);
app.use(markdown({
baseUrl: '/',
root: docDir,
layout: layoutFile,
titleHolder: '<%- locals.title %>',
bodyHolder: '<%- locals.body %>',
indexName: 'readme'
indexName: '_readme'
}));
var locals = {

View File

@@ -14,16 +14,17 @@
* Module dependencies.
*/
var config = require('../config');
var debug = require('debug')('cnpmjs.org:sync:index');
var co = require('co');
var ms = require('ms');
var mail = require('../common/mail');
var util = require('util');
var utility = require('utility');
var debug = require('debug')('cnpmjs.org:sync:index');
var config = require('../config');
var mail = require('../common/mail');
var Total = require('../proxy/total');
var logger = require('../common/logger');
var sync;
var sync = null;
switch (config.syncModel) {
case 'all':
@@ -34,32 +35,45 @@ case 'exist':
break;
}
if (!sync && config.enableCluster) {
console.log('[%s] [sync_worker:%s] no need to sync, exit now', Date(), process.pid);
process.exit(0);
}
console.log('[%s] [sync_worker:%s] syncing with %s mode',
Date(), process.pid, config.syncModel);
//set sync_status = 0 at first
Total.updateSyncStatus(0, utility.noop);
// the same time only sync once
var syncing = false;
function handleSync() {
var handleSync = co(function *() {
debug('mode: %s, syncing: %s', config.syncModel, syncing);
// check sync every one 30 minutes
if (!syncing) {
syncing = true;
debug('start syncing');
sync(function (err, data) {
if (config.debug) {
console.log(err, data);
} else {
sendMailToAdmin(err, data, new Date());
}
syncing = false;
});
var data;
var error;
try {
var data = yield *sync();
} catch (err) {
error = err;
}
if (config.debug) {
error && console.error(error.stack);
data && console.log(data);
} else {
sendMailToAdmin(error, data, new Date());
}
syncing = false;
}
}
});
if (sync) {
handleSync();
setInterval(handleSync, ms('30m'));
setInterval(handleSync, ms(config.syncInterval));
}
function sendMailToAdmin(err, result, syncTime) {
@@ -85,7 +99,7 @@ function sendMailToAdmin(err, result, syncTime) {
'Start sync time is %s.\n %d packges sync failed: %j ...\n %d packages sync successes :%j ...',
syncTime, result.fails.length, result.fails.slice(0, 10),
result.successes.length, result.successes.slice(0, 10));
} else {
} else if (result.successes && result.successes.length) {
subject = 'Sync Finished';
type = 'log';
html = util.format('Sync packages from official registry finished.\n' +
@@ -93,7 +107,7 @@ function sendMailToAdmin(err, result, syncTime) {
syncTime, result.successes.length, result.successes.slice(0, 10));
}
debug('send email with type: %s, subject: %s, html: %s', type, subject, html);
if (type !== 'log') {
if (type && type !== 'log') {
mail[type](to, subject, html, function (err) {
if (err) {
logger.info('send email with type: %s, subject: %s, html: %s', type, subject, html);

View File

@@ -19,18 +19,20 @@ var utility = require('utility');
var Total = require('../proxy/total');
function Status(options) {
this.worker = options.worker;
this.need = options.need;
this.lastSyncModule = '';
this.successes = 0;
this.fails = 0;
this.left = options.need;
}
Status.prototype.log = function (syncDone) {
var params = {
syncStatus: syncDone ? 0 : 1,
need: this.need,
success: this.worker.successes.length,
fail: this.worker.fails.length,
left: this.worker.names.length,
success: this.successes,
fail: this.fails,
left: this.left,
lastSyncModule: this.lastSyncModule,
};
Total.updateSyncNum(params, utility.noop);
@@ -43,19 +45,35 @@ Status.prototype.start = function () {
this.started = true;
//every 30s log it into mysql
this.timer = setInterval(this.log.bind(this), 30000);
this.worker.on('success', function (moduleName) {
debug('sync [%s] success', moduleName);
this.lastSyncModule = moduleName;
}.bind(this));
this.worker.on('end', function () {
this.started = false;
this.log(true);
clearInterval(this.timer);
}.bind(this));
};
Status.init = function (options) {
return new Status(options);
Status.prototype.stop = function () {
this.log(true);
clearInterval(this.timer);
this.timer = null;
this.started = false;
};
Status.init = function (options, worker) {
var status = new Status(options);
status.start();
worker.on('success', function (moduleName) {
debug('sync [%s] success', moduleName);
status.lastSyncModule = moduleName;
status.successes++;
status.left--;
});
worker.on('fail', function () {
status.fails++;
status.left--;
});
worker.on('add', function () {
status.left++;
});
worker.on('end', function () {
status.stop();
});
};
module.exports = Status;

View File

@@ -24,6 +24,7 @@ var Npm = require('../proxy/npm');
var Total = require('../proxy/total');
var SyncModuleWorker = require('../proxy/sync_module_worker');
var Module = require('../proxy/module');
var thunkify = require('thunkify-wrap');
function subtract(subtracter, minuend) {
subtracter = subtracter || [];
@@ -55,109 +56,87 @@ function union(arrOne, arrTwo) {
* when sync from official at the first time
* get all packages by short and restart from last synced module
* @param {String} lastSyncModule
* @param {Function} callback
*/
function getFirstSyncPackages(lastSyncModule, callback) {
Npm.getShort(function (err, pkgs) {
if (err || !lastSyncModule) {
return callback(err, pkgs);
}
// start from last success
var lastIndex = pkgs.indexOf(lastSyncModule);
if (lastIndex > 0) {
pkgs = pkgs.slice(lastIndex);
}
return callback(null, pkgs);
});
function *getFirstSyncPackages(lastSyncModule) {
var pkgs = yield Npm.getShort();
if (!lastSyncModule) {
return pkgs;
}
// start from last success
var lastIndex = pkgs.indexOf(lastSyncModule);
if (lastIndex > 0) {
return pkgs.slice(lastIndex);
}
}
/**
* get all the packages that update time > lastSyncTime
* @param {Number} lastSyncTime
* @param {Function} callback
*/
function getCommonSyncPackages(lastSyncTime, callback) {
Npm.getAllSince(lastSyncTime, function (err, data) {
if (err || !data) {
return callback(err, []);
}
delete data._updated;
return callback(null, Object.keys(data));
});
function *getCommonSyncPackages(lastSyncTime) {
var data = yield Npm.getAllSince(lastSyncTime);
if (!data) {
return [];
}
delete data._updated;
return Object.keys(data);
}
/**
* get all the missing packages
* @param {Function} callback
*/
function getMissPackages(callback) {
var ep = eventproxy.create();
ep.fail(callback);
Npm.getShort(ep.doneLater('allPackages'));
Module.listAllModuleNames(ep.doneLater(function (rows) {
var existPackages = rows.map(function (row) {
return row.name;
});
ep.emit('existPackages', existPackages);
}));
ep.all('allPackages', 'existPackages', function (allPackages, existPackages) {
callback(null, subtract(allPackages, existPackages));
function *getMissPackages(callback) {
var r = yield [Npm.getShort(), Module.listAllModuleNames];
var allPackages = r[0];
var existPackages = r[1].map(function (row) {
return row.name;
});
return subtract(allPackages, existPackages);
}
//only sync not exist once
var syncNotExist = false;
module.exports = function sync(callback) {
var ep = eventproxy.create();
ep.fail(callback);
module.exports = function *sync() {
var syncTime = Date.now();
Total.getTotalInfo(ep.doneLater('totalInfo'));
var info = yield Total.getTotalInfo();
if (!info) {
throw new Error('can not found total info');
}
ep.once('totalInfo', function (info) {
if (!info) {
return callback(new Error('can not found total info'));
}
debug('Last sync time %s', new Date(info.last_sync_time));
// TODO: 记录上次同步的最后一个模块名称
if (!info.last_sync_time) {
debug('First time sync all packages from official registry');
return getFirstSyncPackages(info.last_sync_module, ep.done('syncPackages'));
}
if (syncNotExist) {
getMissPackages(ep.done('missPackages'));
syncNotExist = false;
} else {
ep.emitLater('missPackages', []);
}
getCommonSyncPackages(info.last_sync_time - ms('10m'), ep.doneLater('newestPackages'));
ep.all('missPackages', 'newestPackages', function (missPackages, newestPackages) {
ep.emit('syncPackages', union(missPackages, newestPackages));
});
});
var packages;
debug('Last sync time %s', new Date(info.last_sync_time));
if (!info.last_sync_time) {
debug('First time sync all packages from official registry');
packages = yield getFirstSyncPackages(info.last_sync_module);
} else {
packages = yield getCommonSyncPackages(info.last_sync_time - ms('10m'));
}
ep.once('syncPackages', function (packages) {
packages = packages || [];
debug('Total %d packages to sync', packages.length);
var worker = new SyncModuleWorker({
username: 'admin',
name: packages,
noDep: true,
concurrency: config.syncConcurrency,
});
Status.init({
worker: worker,
need: packages.length
}).start();
worker.start();
worker.once('end', function () {
debug('All packages sync done, successes %d, fails %d',
worker.successes.length, worker.fails.length);
//only when all succss, set last sync time
!worker.fails.length && Total.setLastSyncTime(syncTime, utility.noop);
callback(null, {
successes: worker.successes,
fails: worker.fails
});
});
packages = packages || [];
if (!packages.length) {
debug('no packages need be sync');
return;
}
debug('Total %d packages to sync', packages.length);
var worker = new SyncModuleWorker({
username: 'admin',
name: packages,
noDep: true,
concurrency: config.syncConcurrency,
});
Status.init({need: packages.length}, worker);
worker.start();
var end = thunkify.event(worker);
yield end();
debug('All packages sync done, successes %d, fails %d',
worker.successes.length, worker.fails.length);
//only when all succss, set last sync time
if (!worker.fails.length) {
Total.setLastSyncTime(syncTime, utility.noop);
}
return {
successes: worker.successes,
fails: worker.fails
};
};

View File

@@ -24,6 +24,7 @@ var debug = require('debug')('cnpmjs.org:sync:sync_hot');
var utility = require('utility');
var Status = require('./status');
var ms = require('ms');
var thunkify = require('thunkify-wrap');
function intersection(arrOne, arrTwo) {
arrOne = arrOne || [];
@@ -39,71 +40,67 @@ function intersection(arrOne, arrTwo) {
return results;
}
module.exports = function sync(callback) {
var ep = eventproxy.create();
ep.fail(callback);
module.exports = function *sync() {
var syncTime = Date.now();
Module.listShort(ep.doneLater(function (packages) {
packages = packages.map(function (p) {
return p.name;
});
ep.emit('existPackages', packages);
}));
Total.getTotalInfo(ep.doneLater('totalInfo'));
ep.once('totalInfo', function (info) {
if (!info) {
return callback(new Error('can not found total info'));
}
if (!info.last_exist_sync_time) {
debug('First time sync all packages from official registry');
return Npm.getShort(ep.done(function (pkgs) {
if (!info.last_sync_module) {
return ep.emit('allPackages', pkgs);
}
// start from last success
var lastIndex = pkgs.indexOf(info.last_sync_module);
if (lastIndex > 0) {
pkgs = pkgs.slice(lastIndex);
}
ep.emit('allPackages', pkgs);
}));
}
Npm.getAllSince(info.last_exist_sync_time - ms('10m'), ep.done(function (data) {
if (!data) {
return ep.emit('allPackages', []);
}
if (data._updated) {
syncTime = data._updated;
delete data._updated;
}
return ep.emit('allPackages', Object.keys(data));
}));
var r = yield [Module.listShort(), Total.getTotalInfo()];
var existPackages = r[0].map(function (p) {
return p.name;
});
var info = r[1];
if (!info) {
throw new Error('can not found total info');
}
ep.all('existPackages', 'allPackages', function (existPackages, allPackages) {
var packages = intersection(existPackages, allPackages);
debug('Total %d packages to sync', packages.length);
var worker = new SyncModuleWorker({
username: 'admin',
name: packages,
concurrency: config.syncConcurrency,
});
Status.init({
worker: worker,
need: packages.length
}).start();
worker.start();
worker.once('end', function () {
debug('All packages sync done, successes %d, fails %d',
worker.successes.length, worker.fails.length);
Total.setLastExistSyncTime(syncTime, utility.noop);
callback(null, {
successes: worker.successes,
fails: worker.fails
});
});
var allPackages;
if (!info.last_exist_sync_time) {
debug('First time sync all packages from official registry');
var pkgs = yield Npm.getShort();
if (info.last_sync_module) {
// start from last success
var lastIndex = pkgs.indexOf(info.last_sync_module);
if (lastIndex > 0) {
pkgs = pkgs.slice(lastIndex);
}
}
allPackages = pkgs;
} else {
debug('sync new module from last exist sync time: %s', info.last_sync_time);
var data = yield Npm.getAllSince(info.last_exist_sync_time - ms('10m'));
if (!data) {
allPackages = [];
}
if (data._updated) {
syncTime = data._updated;
delete data._updated;
}
allPackages = Object.keys(data);
}
var packages = intersection(existPackages, allPackages);
if (!packages.length) {
debug('no packages need be sync');
return;
}
debug('Total %d packages to sync', packages.length);
var worker = new SyncModuleWorker({
username: 'admin',
name: packages,
concurrency: config.syncConcurrency
});
Status.init({need: packages.length}, worker);
worker.start();
var end = thunkify.event(worker);
yield end();
debug('All packages sync done, successes %d, fails %d',
worker.successes.length, worker.fails.length);
Total.setLastExistSyncTime(syncTime, utility.noop);
return {
successes: worker.successes,
fails: worker.fails
};
};

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - test/controllers/registry/module.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -31,15 +31,22 @@ var ModuleDeps = require('../../../proxy/module_deps');
var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
describe('controllers/registry/module.test.js', function () {
var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
before(function (done) {
app.listen(0, done);
app.listen(0, function () {
var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
request(app)
.put('/' + pkg.name)
.set('authorization', baseauth)
.send(pkg)
.expect(201, done);
});
});
afterEach(mm.restore);
var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
describe('sync source npm package', function () {
var logId;
it('should put /:name/sync success', function (done) {
@@ -71,24 +78,40 @@ describe('controllers/registry/module.test.js', function () {
it('should return module info and etag', function (done) {
request(app)
.get('/cnpmjs.org')
.get('/mk2testmodule')
.expect('content-type', 'application/json')
.expect(200, function (err, res) {
should.not.exist(err);
// should have etag
res.headers.should.have.property('etag');
etag = res.headers.etag;
etag.should.match(/^"\d{13}"$/);
res.body.should.have.keys('_id', '_rev', 'name', 'description',
'versions', 'dist-tags', 'readme', 'maintainers',
'time', 'author', 'repository', '_attachments');
res.body.name.should.equal('cnpmjs.org');
res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
'time', 'author',
// 'repository',
'_attachments',
'users',
// 'readmeFilename',
// 'homepage',
// 'bugs',
'license');
res.body.name.should.equal('mk2testmodule');
res.body.versions[Object.keys(res.body.versions)[0]]
.dist.tarball.should.include('/mk2testmodule/download');
res.body.time.should.have.property('modified');
res.body.time.modified.should.be.a.String;
res.body.time.should.have.property('created');
res.body.time.created.should.be.a.String;
// should not contains authSession cookie
should.not.exist(res.headers['set-cookie']);
done();
});
});
it('should return module info and gzip when accept-encoding=gzip', function (done) {
request(app)
.get('/cnpmjs.org')
.get('/mk2testmodule')
.set('accept-encoding', 'gzip')
.expect('content-encoding', 'gzip')
.expect(200, function (err, res) {
@@ -99,32 +122,40 @@ describe('controllers/registry/module.test.js', function () {
etag = res.headers.etag;
res.body.should.have.keys('_id', '_rev', 'name', 'description',
'versions', 'dist-tags', 'readme', 'maintainers',
'time', 'author', 'repository', '_attachments');
res.body.name.should.equal('cnpmjs.org');
res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
'time', 'author',
// 'repository',
'_attachments',
'users',
// 'readmeFilename',
// 'homepage', 'bugs',
'license');
res.body.name.should.equal('mk2testmodule');
res.body.versions[Object.keys(res.body.versions)[0]]
.dist.tarball.should.include('/mk2testmodule/download');
should.not.exist(res.headers['set-cookie']);
done();
});
});
it('should 304 when etag match', function (done) {
request(app)
.get('/cnpmjs.org')
.get('/mk2testmodule')
.set('If-None-Match', etag)
.expect(304, done)
.expect(304, done);
});
});
describe('GET /:name/:(version|tag)', function () {
it('should return module@version info', function (done) {
request(app)
.get('/cnpmjs.org/0.2.1')
.get('/mk2testmodule/0.0.1')
.expect(200, function (err, res) {
should.not.exist(err);
var body = res.body;
body.name.should.equal('cnpmjs.org');
body.version.should.equal('0.2.1');
body._id.should.equal('cnpmjs.org@0.2.1');
body.dist.tarball.should.include('cnpmjs.org-0.2.1.tgz');
body.name.should.equal('mk2testmodule');
body.version.should.match(/\d+\.\d+\.\d+/);
body._id.should.match(/mk2testmodule@\d+\.\d+\.\d+/);
body.dist.tarball.should.match(/mk2testmodule\-\d+\.\d+\.\d+\.tgz/);
body.should.have.property('_cnpm_publish_time');
body._cnpm_publish_time.should.be.a.Number;
body.should.have.property('_publish_on_cnpm', true);
@@ -134,29 +165,93 @@ describe('controllers/registry/module.test.js', function () {
it('should return module@tag info', function (done) {
request(app)
.get('/cutter/latest')
.get('/mk2testmodule/latest')
.expect(200, function (err, res) {
should.not.exist(err);
var body = res.body;
body.name.should.equal('cutter');
body.version.should.equal('0.0.3');
body._id.should.equal('cutter@0.0.3');
body.dist.tarball.should.include('/cutter/download/cutter-0.0.3.tgz');
body.name.should.equal('mk2testmodule');
body.version.should.equal('0.0.1');
body._id.should.equal('mk2testmodule@0.0.1');
body.dist.tarball.should.include('/mk2testmodule/download/mk2testmodule-0.0.1.tgz');
done();
});
});
});
it('should get cnpmjs.org@0.2.1 with _publish_on_cnpm=true', function (done) {
describe('PUT /:name/-rev/id update maintainers', function () {
before(function (done) {
request(app)
.get('/cnpmjs.org/0.2.1')
.expect(200, function (err, res) {
should.not.exist(err);
var body = res.body;
body.name.should.equal('cnpmjs.org');
body.version.should.equal('0.2.1');
body._publish_on_cnpm.should.equal(true);
done();
});
.put('/mk2testmodule/-rev/1')
.send({
maintainers: [{
name: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org'
}]
})
.set('authorization', baseauth)
.expect('content-type', 'application/json', done);
});
it('should add new maintainers', function (done) {
request(app)
.put('/mk2testmodule/-rev/1')
.send({
maintainers: [{
name: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org'
}, {
name: 'fengmk2',
email: 'fengmk2@cnpmjs.org'
}]
})
.set('authorization', baseauth)
.expect(201)
.expect('content-type', 'application/json', done);
});
it('should add again new maintainers', function (done) {
request(app)
.put('/mk2testmodule/-rev/1')
.send({
maintainers: [{
name: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org'
}, {
name: 'fengmk2',
email: 'fengmk2@cnpmjs.org'
}]
})
.set('authorization', baseauth)
.expect(201)
.expect('content-type', 'application/json', done);
});
it('should rm maintainers', function (done) {
request(app)
.put('/mk2testmodule/-rev/1')
.send({
maintainers: [{
name: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org'
}]
})
.set('authorization', baseauth)
.expect(201)
.expect('content-type', 'application/json', done);
});
it('should rm again maintainers', function (done) {
request(app)
.put('/mk2testmodule/-rev/1')
.send({
maintainers: [{
name: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org'
}]
})
.set('authorization', baseauth)
.expect(201)
.expect('content-type', 'application/json', done);
});
});
@@ -214,7 +309,7 @@ describe('controllers/registry/module.test.js', function () {
.expect(201, done);
});
it('should try to add return 403 when not module user and only next module exists',
it.skip('should try to add return 403 when not module user and only next module exists',
function (done) {
mm(config, 'enablePrivate', false);
request(app)
@@ -237,7 +332,7 @@ describe('controllers/registry/module.test.js', function () {
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('_id', '_rev', 'name', 'description', 'versions', 'dist-tags',
'readme', 'maintainers', 'time', '_attachments');
'readme', 'maintainers', 'time', '_attachments', 'users');
res.body.versions.should.eql({});
res.body.time.should.eql({});
res.body['dist-tags'].should.eql({});
@@ -466,7 +561,6 @@ describe('controllers/registry/module.test.js', function () {
res.body._updated.should.be.a.Number;
var keys = Object.keys(res.body);
keys.length.should.be.above(1);
res.body[keys[1]].dist.tarball.should.include('/download/');
done();
});
});
@@ -555,7 +649,7 @@ describe('controllers/registry/module.test.js', function () {
request(app)
.get('/cutter/download/cutter-0.0.2.tgz')
.expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
.expect(302, done)
.expect(302, done);
});
});
@@ -591,7 +685,66 @@ describe('controllers/registry/module.test.js', function () {
});
});
describe('DELETE /:name/-rev/:rev', function (done) {
describe('PUT /:name/:tag', function () {
it('should create new tag ok', function (done) {
request(app)
.put('/testputmodule/newtag')
.set('content-type', 'application/json')
.set('authorization', baseauth)
.send('"0.1.9"')
.expect(201, done);
});
it('should override exist tag ok', function (done) {
request(app)
.put('/testputmodule/newtag')
.set('content-type', 'application/json')
.set('authorization', baseauth)
.send('"0.1.9"')
.expect(201, done);
});
it('should tag invalid version 403', function (done) {
request(app)
.put('/testputmodule/newtag')
.set('content-type', 'application/json')
.set('authorization', baseauth)
.send('"hello"')
.expect(403)
.expect({
error: 'forbidden',
reason: 'setting tag newtag to invalid version: hello: testputmodule/newtag'
}, done);
});
it('should tag not eixst version 403', function (done) {
request(app)
.put('/testputmodule/newtag')
.set('content-type', 'application/json')
.set('authorization', baseauth)
.send('"5.0.0"')
.expect(403)
.expect({
error: 'forbidden',
reason: 'setting tag newtag to unknown version: 5.0.0: testputmodule/newtag'
}, done);
});
it('should tag permission 403', function (done) {
request(app)
.put('/testputmodule/newtag')
.set('content-type', 'application/json')
.set('authorization', baseauthOther)
.send('"0.1.9"')
.expect(403)
.expect({
error: 'forbidden',
reason: 'no permission to modify testputmodule'
}, done);
});
});
describe('DELETE /:name/-rev/:rev', function () {
var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
var lastRev;
@@ -623,7 +776,11 @@ describe('controllers/registry/module.test.js', function () {
request(app)
.del('/testputmodule/-rev/' + lastRev)
.set('authorization', baseauth)
.expect(200, done);
.expect(200, function (err, res) {
should.not.exist(err);
should.not.exist(res.headers['set-cookie']);
done();
});
});
});
});

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - test/controllers/registry/user.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -35,11 +35,26 @@ describe('controllers/registry/user.test.js', function () {
describe('GET /-/user/org.couchdb.user:name', function () {
it('should return user info', function (done) {
request(app)
.get('/-/user/org.couchdb.user:cnpmjstest1')
.get('/-/user/org.couchdb.user:cnpmjstest10')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type', 'roles', 'date');
res.body.name.should.equal('cnpmjstest1');
res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type',
'_cnpm_meta', 'roles', 'date');
res.body.name.should.equal('cnpmjstest10');
done();
});
});
it.skip('should return npm user info', function (done) {
request(app)
.get('/-/user/org.couchdb.user:fengmk2')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.name.should.equal('fengmk2');
res.body.github.should.equal('fengmk2');
res.body._cnpm_meta.should.have.keys('id', 'npm_user', 'gmt_create',
'gmt_modified', 'admin');
res.body._cnpm_meta.admin.should.equal(true);
done();
});
});
@@ -149,7 +164,12 @@ describe('controllers/registry/user.test.js', function () {
ok: true,
name: 'name',
roles: []
}, done);
}, function (err, res) {
should.not.exist(err);
should.exist(res.headers['set-cookie']);
res.headers['set-cookie'].join(';').should.include('AuthSession=');
done();
});
});
});
@@ -171,19 +191,26 @@ describe('controllers/registry/user.test.js', function () {
mm.error(user, 'update', 'mock error');
request(app)
.put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
.send({
name: 'cnpmjstest10',
password: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org'
})
.set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
.expect(500, done);
});
it('should 409 when req.body.rev error', function (done) {
it('should 201 when req.body.rev error', function (done) {
request(app)
.put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
.set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
.send({
name: 'cnpmjstest10',
password: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org',
rev: '1-123'
})
.expect(409, done);
.expect(201, done);
});
it('should 201 update ok', function (done) {
@@ -192,6 +219,8 @@ describe('controllers/registry/user.test.js', function () {
.put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
.set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
.send({
name: 'cnpmjstest10',
password: 'cnpmjstest10',
email: 'cnpmjstest10@cnpmjs.org',
rev: '1-123'
})

View File

@@ -1,4 +1,4 @@
/*!
/**!
* cnpmjs.org - test/controllers/sync.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
@@ -17,12 +17,12 @@
var request = require('supertest');
var should = require('should');
var registryApp = require('../../servers/registry');
var webApp = require('../../servers/web');
var pedding = require('pedding');
var mm = require('mm');
var Npm = require('../../proxy/npm');
var path = require('path');
var Npm = require('../../proxy/npm');
var registryApp = require('../../servers/registry');
var webApp = require('../../servers/web');
describe('controllers/sync.test.js', function () {
before(function (done) {
@@ -45,8 +45,6 @@ describe('controllers/sync.test.js', function () {
request(registryApp)
.del('/utility/-rev/123')
.set('authorization', baseauth)
// .expect(200)
// .expect({ok: true})
.end(function (err, res) {
should.not.exist(err);
@@ -59,20 +57,6 @@ describe('controllers/sync.test.js', function () {
res.body.should.have.keys('ok', 'logId');
logIdRegistry = res.body.logId;
done();
// setTimeout(function () {
// request(registryApp)
// .get('/utility')
// .expect(200)
// .end(function (err, res) {
// should.not.exist(err);
// Object.keys(res.body.versions).length.should.above(0);
// for (var v in res.body.versions) {
// var pkg = res.body.versions[v];
// pkg.should.have.property('_publish_on_cnpm', true);
// }
// done();
// });
// }, 5000);
});
});
});
@@ -88,9 +72,20 @@ describe('controllers/sync.test.js', function () {
}, done);
});
it('should sync success', function (done) {
it('should sync through web success', function (done) {
mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
request(webApp)
.put('/sync/utility')
.end(function (err, res) {
should.not.exist(err);
res.body.should.have.keys('ok', 'logId');
logIdWeb = res.body.logId;
done();
});
});
it('should sync through registry success', function (done) {
mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
done = pedding(2, done);
request(registryApp)
.put('/utility/sync')
.set('authorization', baseauth)
@@ -100,14 +95,6 @@ describe('controllers/sync.test.js', function () {
logIdRegistry = res.body.logId;
done();
});
request(webApp)
.put('/sync/utility')
.end(function (err, res) {
should.not.exist(err);
res.body.should.have.keys('ok', 'logId');
logIdWeb = res.body.logId;
done();
});
});
it('should get sync log', function (done) {

View File

@@ -0,0 +1,37 @@
/**!
* cnpmjs.org - test/controllers/web/dist.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var request = require('supertest');
var app = require('../../../servers/web');
describe('controllers/web/dist.test.js', function () {
before(function (done) {
app.listen(0, done);
});
after(function (done) {
app.close(done);
});
describe('GET /dist', function (done) {
it('should 302 to config.disturl', function (done) {
request(app)
.get('/dist')
.expect('Location', 'http://dist.u.qiniudn.com/')
.expect(302, done);
});
});
});

View File

@@ -17,14 +17,30 @@
var should = require('should');
var request = require('supertest');
var mm = require('mm');
var path = require('path');
var mysql = require('../../../common/mysql');
var app = require('../../../servers/web');
var registry = require('../../../servers/registry');
var pkg = require('../../../controllers/web/package');
var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
describe('controllers/web/package.test.js', function () {
var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
before(function (done) {
app.listen(0, done);
registry.listen(0, function () {
var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
request(registry)
.put('/' + pkg.name)
.set('authorization', baseauth)
.send(pkg)
.expect(201, function () {
app.listen(0, done);
});
});
});
after(function (done) {
app.close(done);
});
@@ -32,9 +48,10 @@ describe('controllers/web/package.test.js', function () {
afterEach(mm.restore);
describe('GET /_list/search/search', function () {
it('should search with "c"', function (done) {
it('should search with "m"', function (done) {
request(app)
.get('/_list/search/search?startkey="c"&limit=2')
.get('/_list/search/search?startkey="m"&limit=2')
.expect('content-type', 'application/json')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('rows');
@@ -47,9 +64,9 @@ describe('controllers/web/package.test.js', function () {
});
});
it('should search with c', function (done) {
it('should search with m', function (done) {
request(app)
.get('/_list/search/search?startkey=c&limit=2')
.get('/_list/search/search?startkey=m&limit=2')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('rows');
@@ -76,14 +93,16 @@ describe('controllers/web/package.test.js', function () {
describe('GET /package/:name', function (done) {
it('should get 200', function (done) {
request(app)
.get('/package/cutter')
.get('/package/mk2testmodule')
.expect(200)
.expect('content-encoding', 'gzip')
.expect('content-type', 'text/html; charset=utf-8')
.expect(/<div id="package">/)
.expect(/<th>Maintainers<\/th>/)
.expect(/<th>Version<\/th>/, function (err, res) {
should.not.exist(err);
res.should.have.header('etag');
res.text.should.include('<meta charset="utf-8">');
done();
});
});
@@ -98,7 +117,7 @@ describe('controllers/web/package.test.js', function () {
describe('GET /package/:name/:version', function (done) {
it('should 200 when get by version', function (done) {
request(app)
.get('/package/cutter/0.0.2')
.get('/package/mk2testmodule/0.0.1')
.expect(200)
.expect(/<div id="package">/)
.expect(/<th>Maintainers<\/th>/)
@@ -107,21 +126,22 @@ describe('controllers/web/package.test.js', function () {
it('should 200 when get by tag', function (done) {
request(app)
.get('/package/cutter/latest')
.get('/package/mk2testmodule/latest')
.expect(200)
.expect(/<div id="package">/)
.expect(/<th>Maintainers<\/th>/)
.expect(/<th>Version<\/th>/, done);
});
it('should 404 when get by version not exist', function (done) {
request(app)
.get('/package/cutter/1.1.2')
.get('/package/mk2testmodule/1.1.2')
.expect(404, done);
});
it('should 404 when get by tag', function (done) {
request(app)
.get('/package/cutter/notexisttag')
.get('/package/mk2testmodule/notexisttag')
.expect(404, done);
});
});
@@ -129,14 +149,14 @@ describe('controllers/web/package.test.js', function () {
describe('GET /browse/keyword/:word', function () {
it('should list by keyword ok', function (done) {
request(app)
.get('/browse/keyword/cnpm')
.get('/browse/keyword/mk2testmodule')
.expect(200)
.expect(/Packages match/, done);
});
it('should list by keyword with json ok', function (done) {
request(app)
.get('/browse/keyword/cnpm?type=json')
.get('/browse/keyword/mk2testmodule?type=json')
.expect(200)
.expect('content-type', 'application/json; charset=utf-8', done);
});
@@ -167,7 +187,7 @@ describe('controllers/web/package.test.js', function () {
url: 'http://opensource.org/licenses/MIT'
});
var p = {license: ['http://foo/MIT']};
p = {license: ['http://foo/MIT']};
pkg.setLicense(p);
p.license.should.have.keys('name', 'url');
p.license.should.eql({
@@ -175,7 +195,7 @@ describe('controllers/web/package.test.js', function () {
url: 'http://foo/MIT'
});
var p = {license: {name: 'mit', url: 'http://foo/mit'}};
p = {license: {name: 'mit', url: 'http://foo/mit'}};
pkg.setLicense(p);
p.license.should.have.keys('name', 'url');
p.license.should.eql({

View File

@@ -29,8 +29,9 @@ describe('controllers/web/user.test.js', function () {
describe('GET /~:name', function (done) {
it('should get 200', function (done) {
request(app)
.get('/~dead_horse')
.get('/~cnpmjstest10')
.expect(200)
.expect('content-type', 'text/html; charset=utf-8')
.expect(/<div id="profile">/)
.expect(/Packages by /, done);
});
@@ -43,7 +44,7 @@ describe('controllers/web/user.test.js', function () {
it('should get not eixst user but have modules 200', function (done) {
request(app)
.get('/~tjholowaychuk')
.get('/~cnpmjstest101')
.expect(200)
.expect(/<div id="profile">/)
.expect(/Packages by/, done);

1
test/fixtures/cnpmtest-package.json vendored Normal file
View File

@@ -0,0 +1 @@
{"_id":"cnpmtest-package","_rev":"382002","name":"cnpmtest-package","description":"cnpmtest-package","dist-tags":{"latest":"0.0.0"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"time":{"modified":"2014-04-09T10:54:45.000Z","created":"2014-04-09T10:54:39.425Z","0.0.0":"2014-04-09T10:54:39.425Z"},"users":{},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"versions":{"0.0.0":{"name":"cnpmtest-package","version":"0.0.0","description":"cnpmtest-package","main":"index.js","scripts":{"test":"make test-all"},"config":{"cov":{"threshold":100}},"dependencies":{},"devDependencies":{"autod":"*","contributors":"*","should":"*","jshint":"*","cov":"*","istanbul-harmony":"*","mocha":"*"},"homepage":"https://github.com/{{group}}/cnpmtest-package","repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"keywords":["cnpmtest-package"],"engines":{"node":">= 0.10.0"},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"license":"MIT","contributors":[],"_id":"cnpmtest-package@0.0.0","dist":{"shasum":"81ea57c24a7f9f6e7263385116fc433f0ad9e179","size":1839,"noattachment":false,"tarball":"http://r.cnpmjs.org/cnpmtest-package/download/cnpmtest-package-0.0.0.tgz"},"_from":".","_npmVersion":"1.4.6","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"directories":{},"publish_time":1397040879425,"_cnpm_publish_time":1397040879425}},"readme":"cnpmtest-package\n=======\n\n[![Build Status](https://secure.travis-ci.org/{{group}}/cnpmtest-package.png)](http://travis-ci.org/{{group}}/cnpmtest-package)\n[![Dependency Status](https://gemnasium.com/{{group}}/cnpmtest-package.png)](https://gemnasium.com/{{group}}/cnpmtest-package)\n\n[![NPM](https://nodei.co/npm/cnpmtest-package.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmtest-package/)\n\n![logo](https://raw.github.com/{{group}}/cnpmtest-package/master/logo.png)\n\ncnpmtest-package desc\n\n## Install\n\n```bash\n$ npm install cnpmtest-package\n```\n\n## Usage\n\n```js\nvar cnpmtest-package = require('cnpmtest-package');\n\ncnpmtest-package.foo(function (err) {\n\n});\n```\n\n## License\n\n(The MIT License)\n\nCopyright (c) 2014 fengmk2 &lt;fengmk2@gmail.com&gt; and other contributors\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n'Software'), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n","_attachments":{},"homepage":"https://github.com/{{group}}/cnpmtest-package","bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"license":"MIT"}

1
test/fixtures/fengmk2.json vendored Normal file
View File

@@ -0,0 +1 @@
{"_id":"org.couchdb.user:fengmk2","_rev":"25-ba29aa975022944b4ba547ab11ce3af5","name":"fengmk2","email":"fengmk2@gmail.com","type":"user","roles":[],"date":"2014-03-15T08:06:18.870Z","fullname":"fengmk2","avatar":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=50&d=retro","freenode":"","github":"fengmk2","homepage":"http://fengmk2.github.com","twitter":"fengmk2","avatarMedium":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=100&d=retro","avatarLarge":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=496&d=retro","fields":[{"name":"fullname","value":"fengmk2","title":"Full Name","show":"fengmk2"},{"name":"email","value":"fengmk2@gmail.com","title":"Email","show":"<a href=\"mailto:fengmk2@gmail.com\">fengmk2@gmail.com</a>"},{"name":"github","value":"fengmk2","title":"Github","show":"<a rel=\"me\" href=\"https://github.com/fengmk2\">fengmk2</a>"},{"name":"twitter","value":"fengmk2","title":"Twitter","show":"<a rel=\"me\" href=\"https://twitter.com/fengmk2\">@fengmk2</a>"},{"name":"appdotnet","value":"","title":"App.net","show":""},{"name":"homepage","value":"http://fengmk2.github.com","title":"Homepage","show":"<a rel=\"me\" href=\"http://fengmk2.github.com/\">http://fengmk2.github.com</a>"},{"name":"freenode","value":"","title":"IRC Handle","show":""}],"appdotnet":"fengmk2"}

41
test/init.js Normal file
View File

@@ -0,0 +1,41 @@
/**!
* cnpmjs.org - test/init.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var crypto = require('crypto');
var utility = require('utility');
var User = require('../proxy/user');
var usernames = [
'cnpmjstest101',
'cnpmjstest10'
];
usernames.forEach(function (name) {
var user = {
name: name,
email: 'fengmk2@gmail.com',
// password: 'cnpmjstest10',
ip: '127.0.0.1'
};
user.salt = crypto.randomBytes(30).toString('hex');
user.password_sha = utility.sha1(user.name + user.salt);
User.add(user, function (err, result) {
if (err) {
throw err;
}
});
});

View File

@@ -34,7 +34,7 @@ describe('middleware/auth.test.js', function () {
describe('auth()', function () {
it('should pass if no authorization', function (done) {
request(app)
.get('/-/user/org.couchdb.user:cnpmjstest1')
.get('/-/user/org.couchdb.user:cnpmjstest10')
.expect(200, done);
});

View File

@@ -0,0 +1,60 @@
/**!
* cnpmjs.org - test/middleware/static.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var request = require('supertest');
var registry = require('../../servers/registry');
var web = require('../../servers/registry');
describe('middleware/static.test.js', function () {
before(function (done) {
registry = registry.listen(0, function () {
web = web.listen(0, done);
});
});
describe('registry', function () {
it('should /favicon.ico rewrite to /favicon.png', function (done) {
request(registry)
.get('/favicon.ico')
// .expect('content-type', 'image/png')
.expect(200, done);
});
it('should 200 /favicon.png', function (done) {
request(registry)
.get('/favicon.png')
.expect('content-type', 'image/png')
.expect(200, done);
});
});
describe('web', function () {
it('should /favicon.ico rewrite to /favicon.png', function (done) {
request(registry)
.get('/favicon.ico')
// .expect('content-type', 'image/png')
.expect(200, done);
});
it('should 200 /favicon.png', function (done) {
request(registry)
.get('/favicon.png')
.expect('content-type', 'image/png')
.expect(200, done);
});
});
});

View File

@@ -0,0 +1,63 @@
/**!
* cnpmjs.org - test/middleware/web_not_found.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var request = require('supertest');
var app = require('../../servers/web');
describe('middleware/web_not_found.test.js', function () {
before(function (done) {
app.listen(0, done);
});
after(function (done) {
app.close(done);
});
describe('web_not_found()', function () {
it('should redirect /mk2testmodule to /package/mk2testmodule', function (done) {
request(app)
.get('/mk2testmodule')
.expect('Location', '/package/mk2testmodule')
.expect(302, done);
});
it('should redirect /mk2testmodule/ to /package/mk2testmodule', function (done) {
request(app)
.get('/mk2testmodule/')
.expect('Location', '/package/mk2testmodule')
.expect(302, done);
});
it('should 404 /~byte', function (done) {
request(app)
.get('/~byte')
.expect(404, done);
});
it('should 200 /package/mk2testmodule', function (done) {
request(app)
.get('/package/mk2testmodule')
.expect(200, done);
});
it('should 404 /package/byte404', function (done) {
request(app)
.get('/package/byte404')
.expect(404, done);
});
});
});

View File

@@ -38,8 +38,17 @@ describe('proxy/module.test.js', function () {
});
describe('search()', function () {
it('should search modules', function (done) {
Module.search('as', function (err, data) {
before(function (done) {
Module.addKeywords('aaaa', 'mock aaaaaa', ['aa', 'bb', 'cc'], function (err, results) {
should.not.exist(err);
results.should.be.an.Array;
results.should.length(3);
done();
});
});
it.skip('should search modules', function (done) {
Module.search('mock', function (err, data) {
should.not.exist(err);
data.should.have.keys('keywordMatchs', 'searchMatchs');
data.searchMatchs.length.should.above(0);
@@ -74,13 +83,8 @@ describe('proxy/module.test.js', function () {
});
});
var mockName = 'aa' + Date.now();
describe('addKeywords()', function () {
var mockName = 'aa' + Date.now();
after(function (done) {
mysql.query('DELETE FROM module_keyword WHERE name=?', [mockName], done);
});
it('should add diff keywords to module', function (done) {
Module.addKeywords(mockName, mockName, ['aa', 'bb', 'cc'], function (err, results) {
should.not.exist(err);
@@ -91,10 +95,11 @@ describe('proxy/module.test.js', function () {
});
it('should add same keywords to module', function (done) {
Module.addKeywords('aa', 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
Module.addKeywords(mockName, 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
should.not.exist(err);
results.should.be.an.Array;
results.should.length(0);
results.should.length(3);
// results.should.length(0);
done();
});
});
@@ -102,7 +107,7 @@ describe('proxy/module.test.js', function () {
describe('getKeywords()', function () {
it('should get aa module keywords', function (done) {
Module.getKeywords('aa', function (err, keywords) {
Module.getKeywords(mockName, function (err, keywords) {
should.not.exist(err);
keywords.should.eql(['aa', 'bb', 'cc']);
done();
@@ -158,7 +163,7 @@ describe('proxy/module.test.js', function () {
should.not.exist(err);
data.package.readme.should.equal('test');
done();
})
});
});
});
});

View File

@@ -0,0 +1,51 @@
/**!
* cnpmjs.org - test/proxy/module_star.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var should = require('should');
var Star = require('../../proxy/module_star');
describe('proxy/module_star.test.js', function () {
before(function *() {
yield Star.remove('testmodule', 'fengmk2');
yield Star.remove('testmodule', 'mk1');
yield Star.remove('testmodule', 'mk2');
});
it('should add a star', function *() {
yield Star.add('testmodule', 'fengmk2');
// again should be ok
yield Star.add('testmodule', 'fengmk2');
yield Star.add('testmodule', 'fengmk2');
});
it('should get all star users', function *() {
yield Star.add('testmodule', 'fengmk2');
yield Star.add('testmodule', 'mk1');
yield Star.add('testmodule', 'mk2');
var rows = yield Star.listUsers('testmodule');
rows.should.containDeep(['fengmk2', 'mk1', 'mk2']);
});
it('should get user all star modules', function *() {
yield Star.add('testmodule', 'fengmk2');
yield Star.add('testmodule1', 'fengmk2');
yield Star.add('testmodule2', 'fengmk2');
var rows = yield Star.listUserModules('fengmk2');
rows.should.containDeep(['testmodule', 'testmodule1', 'testmodule2']);
});
});

View File

@@ -18,6 +18,7 @@ var should = require('should');
var mm = require('mm');
var fs = require('fs');
var path = require('path');
var ChunkStream = require('chunkstream');
var npm = require('../../proxy/npm');
var fixtures = path.join(path.dirname(__dirname), 'fixtures');
@@ -25,42 +26,36 @@ var fixtures = path.join(path.dirname(__dirname), 'fixtures');
describe('proxy/npm.test.js', function () {
afterEach(mm.restore);
it('should return a module info from source npm', function (done) {
npm.get('pedding', function (err, data) {
should.not.exist(err);
should.exist(data);
data.name.should.equal('pedding');
done();
});
it('should return a module info from source npm', function *() {
var data = yield npm.get('pedding');
data.name.should.equal('pedding');
});
it('should return null when module not exist', function (done) {
npm.get('pedding-not-exists', function (err, data) {
should.not.exist(err);
should.not.exist(data);
done();
});
it('should return null when module not exist', function *() {
var data = yield npm.get('pedding-not-exists');
should.not.exist(data);
});
it('should return error when http error', function (done) {
mm.http.request(/\//, '{');
npm.get('pedding-not-exists', function (err, data) {
should.exist(err);
it('should return error when http error', function *() {
mm.http.request(/\//, new ChunkStream(['{']));
try {
var data = yield npm.get('pedding-not-exists');
throw new Error('should not run this');
} catch (err) {
err.name.should.equal('JSONResponseFormatError');
should.not.exist(data);
done();
});
}
});
it('should return ServerError when http 500 response', function (done) {
var content = fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8');
it('should return ServerError when http 500 response', function *() {
var content = fs.createReadStream(path.join(fixtures, '500.txt'));
mm.http.request(/\//, content, { statusCode: 500 });
// http://registry.npmjs.org/octopie
npm.get('octopie', function (err, data) {
should.exist(err);
try {
var data = yield npm.get('octopie');
throw new Error('should not run this');
} catch (err) {
err.name.should.equal('NPMServerError');
err.message.should.equal('Status 500, ' + content);
done();
});
err.message.should.equal('Status 500, ' + fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8'));
}
});
});

View File

@@ -22,14 +22,14 @@ var Log = require('../../proxy/module_log');
describe('proxy/sync_module_worker.test.js', function () {
it('should start a sync worker', function (done) {
Log.create({
name: 'cnpmjs.org',
name: 'mk2testmodule',
username: 'fengmk2',
}, function (err, result) {
should.not.exist(err);
result.id.should.above(0);
var worker = new SyncModuleWorker({
logId: result.id,
name: 'cnpmjs.org',
name: 'mk2testmodule',
username: 'fengmk2'
});
@@ -40,21 +40,23 @@ describe('proxy/sync_module_worker.test.js', function () {
it('should start a sync worker with names and noDep', function (done) {
var worker = new SyncModuleWorker({
name: ['cnpmjs.org', 'cutter'],
name: ['mk2testmodule'],
noDep: true,
username: 'fengmk2'
});
worker.start();
worker.on('end', function () {
worker.successes.concat(worker.fails).should.eql(['cnpmjs.org', 'cutter']);
var names = worker.successes.concat(worker.fails);
names.sort();
names.should.eql(['mk2testmodule']);
done();
});
});
it('should start a sync worker with names', function (done) {
var worker = new SyncModuleWorker({
name: ['cnpmjs.org', 'cutter'],
name: ['mk2testmodule'],
username: 'fengmk2'
});

View File

@@ -6,6 +6,7 @@
*
* Authors:
* dead_horse <dead_horse@qq.com> (http://deadhorse.me)
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
*/
'use strict';
@@ -14,10 +15,14 @@
* Module dependencies.
*/
var mysql = require('../../common/mysql');
var should = require('should');
var user = require('../../proxy/user');
var mm = require('mm');
var path = require('path');
var fs = require('fs');
var mysql = require('../../common/mysql');
var user = require('../../proxy/user');
var fixtures = path.join(path.dirname(__dirname), 'fixtures');
var mockUser = {
name: 'mockuser',
@@ -49,7 +54,9 @@ describe('proxy/user.test.js', function () {
it('should get user ok', function (done) {
user.get('mockuser', function (err, data) {
should.not.exist(err);
data.should.have.keys('id', 'rev', 'name', 'email', 'salt', 'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
data.should.have.keys('id', 'rev', 'name', 'email', 'salt',
'json', 'npm_user',
'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
done();
});
});
@@ -68,7 +75,9 @@ describe('proxy/user.test.js', function () {
it('should auth user ok', function (done) {
user.auth(mockUser.name, mockUser.password, function (err, data) {
should.not.exist(err);
data.should.have.keys('id', 'rev', 'name', 'email', 'salt', 'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
data.should.have.keys('id', 'rev', 'name', 'email', 'salt',
'json', 'npm_user',
'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
done();
});
});
@@ -118,11 +127,12 @@ describe('proxy/user.test.js', function () {
describe('update()', function () {
before(initUser);
it('should update ok', function (done) {
user.update(mockUser, function (err, data) {
should.not.exist(err);
should.exist(data);
data.should.have.keys(['rev']);
data.should.have.keys('rev', 'result');
done();
});
});
@@ -131,7 +141,8 @@ describe('proxy/user.test.js', function () {
mockUser.rev = '1-error';
user.update(mockUser, function (err, data) {
should.not.exist(err);
should.not.exist(data);
should.exist(data);
data.result.affectedRows.should.equal(0);
done();
});
});
@@ -153,4 +164,32 @@ describe('proxy/user.test.js', function () {
});
});
});
describe('saveNpmUser()', function () {
var existUser = JSON.parse(fs.readFileSync(path.join(fixtures, 'fengmk2.json')));
var notExistUser = JSON.parse(fs.readFileSync(path.join(fixtures, 'fengmk2.json')));
notExistUser.name = 'fengmk2-not-exists';
before(function *() {
yield mysql.query('delete from user where name=?', [notExistUser.name]);
});
it('should save npm user to exists user', function *() {
yield user.saveNpmUser(existUser);
var r = yield mysql.queryOne('select rev, json, npm_user from user where name=?', existUser.name);
should.exist(r);
// r.npm_user.should.equal(0);
r.rev.should.equal(existUser._rev);
JSON.parse(r.json).should.eql(existUser);
});
it('should save npm user to not exists user and create it', function *() {
yield user.saveNpmUser(notExistUser);
var r = yield mysql.queryOne('select name, json, npm_user from user where name=?', notExistUser.name);
r.name.should.equal(notExistUser.name);
should.exist(r);
r.npm_user.should.equal(1);
JSON.parse(r.json).should.eql(notExistUser);
});
});
});

View File

@@ -17,21 +17,27 @@
var SyncModuleWorker = require('../proxy/sync_module_worker');
var mysql = require('../common/mysql');
var Log = require('../proxy/module_log');
var config = require('../config');
var name = process.argv[2] || 'address,pedding';
var names = name.split(',');
config.sourceNpmRegistry = 'http://r.cnpmjs.org';
var names = process.argv[2] || 'byte';
names = names.split(',');
Log.create({
name: names[0],
username: 'fengmk2',
}, function (err, result) {
if (err) {
throw err;
}
var worker = new SyncModuleWorker({
logId: result.id,
name: names,
username: 'fengmk2',
concurrency: names.length,
noDep: true,
publish: true,
// noDep: true,
// publish: true,
});
worker.start();

View File

@@ -1,5 +1,5 @@
/*!
* cnpmjs.org - test/sync/sync_all.js
* cnpmjs.org - test/sync/sync_all.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
@@ -20,39 +20,33 @@ var Total = require('../../proxy/total');
var should = require('should');
var Module = require('../../proxy/module');
describe('sync/sync_all.js', function () {
describe('sync/sync_all.test.js', function () {
describe('sync()', function () {
afterEach(mm.restore);
it('should sync first time ok', function (done) {
mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
it('should sync first time ok', function *() {
mm.data(Npm, 'getShort', ['mk2testmodule', 'mk2testmodule-not-exists']);
mm.data(Total, 'getTotalInfo', {last_sync_time: 0});
sync(function (err, data) {
should.not.exist(err);
data.successes.should.eql(['cnpmjs.org', 'cutter']);
mm.restore();
Total.getTotalInfo(function (err, result) {
should.not.exist(err);
result.last_sync_module.should.equal('cutter');
done();
});
});
var data = yield sync;
data.successes.should.eql(['mk2testmodule', 'mk2testmodule-not-exists']);
mm.restore();
var result = yield Total.getTotalInfo();
should.exist(result);
result.last_sync_module.should.equal('mk2testmodule-not-exists');
});
it('should sync common ok', function (done) {
it('should sync common ok', function *() {
mm.data(Npm, 'getAllSince', {
_updated: Date.now(),
'cnpmjs.org': {},
cutter: {}
'mk2testmodule': {},
// cutter: {}
});
mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
mm.data(Npm, 'getShort', ['mk2testmodule']);
mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
sync(function (err, data) {
should.not.exist(err);
data.successes.should.eql(['cnpmjs.org', 'cutter']);
done();
});
mm.data(Module, 'listAllModuleNames', [{name: 'mk2testmodule'}]);
var data = yield sync;
data.successes.should.eql(['mk2testmodule']);
mm.restore();
});
});
});

View File

@@ -1,5 +1,5 @@
/*!
* cnpmjs.org - test/sync/sync_exist.js
* cnpmjs.org - test/sync/sync_exist.test.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
@@ -13,38 +13,32 @@
/**
* Module dependencies.
*/
var sync = require('../../sync/sync_exist');
var should = require('should');
var mm = require('mm');
var sync = require('../../sync/sync_exist');
var Npm = require('../../proxy/npm');
var Total = require('../../proxy/total');
var should = require('should');
describe('sync/sync_exist.js', function () {
describe('sync/sync_exist.test.js', function () {
describe('sync()', function () {
afterEach(mm.restore);
it('should sync first time ok', function (done) {
mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
it('should sync first time ok', function *() {
mm.data(Npm, 'getShort', ['mk2testmodule']);
mm.data(Total, 'getTotalInfo', {last_exist_sync_time: 0});
sync(function (err, data) {
should.not.exist(err);
data.successes.should.eql(['cnpmjs.org', 'cutter']);
done();
});
var data = yield sync();
data.successes.should.eql(['mk2testmodule']);
});
it('should sync common ok', function (done) {
it('should sync common ok', function *() {
mm.data(Npm, 'getAllSince', {
_updated: Date.now(),
'cnpmjs.org': {},
cutter: {}
'mk2testmodule': {},
});
mm.data(Total, 'getTotalInfo', {last_exist_sync_time: Date.now()});
sync(function (err, data) {
should.not.exist(err);
data.successes.should.eql(['cnpmjs.org', 'cutter']);
done();
});
var data = yield sync();
data.successes.should.eql(['mk2testmodule']);
});
});
});

View File

@@ -1,9 +1,10 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title><%- locals.title %></title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="shortcut icon" href="/public/favicon.png">
<link rel="shortcut icon" href="/favicon.png">
<!-- Bootstrap -->
<link href="http://cdn.staticfile.org/twitter-bootstrap/3.0.0-rc2/css/bootstrap.min.css" rel="stylesheet" media="screen">
<link href="http://cdn.staticfile.org/prettify/r298/prettify.min.css" rel="stylesheet" media="screen">

View File

@@ -85,6 +85,18 @@
<% } %>
</td>
</tr>
<% if (package.engines) {%>
<tr>
<th>Engines</th>
<td>
<ul>
<% for (var k in package.engines) { %>
<li><%= k %>: <%= package.engines[k] %></li>
<% } %>
</ul>
</td>
</tr>
<% } %>
<% if (package.license) { %>
<tr>
<th>License</th>
@@ -188,27 +200,27 @@
</td>
</tr>
<% if (package.users) {
var starredBy = package.starredBy
var l = starredBy.length
<% if (package.users.length > 0) {
var users = package.users
var l = users.length
%>
<tr>
<th>Starred by<%= l > 0 ? ' (' + l + ')' : '' %></th>
<th>Starred by (<%= l %>)</th>
<td>
<%
var max = 20
if (l > max) {
starredBy = starredBy.sort(function (a, b) {
users = users.sort(function (a, b) {
return Math.random() * 2 - 1
}).slice(0, max)
}
starredBy.forEach(function (user, i) {
users.forEach(function (user, i) {
if (i > 0) { %>, <% }
%><a href="/~<%= user %>"><%= user %></a><%
})
if (l > max) {
%><br><a href="/browse/star/<%= package.name %>">and
<%= (l-max) %> more</a><%
<%= (l - max) %> more</a><%
}
%>
</td>
@@ -240,6 +252,7 @@
<a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
<%= package.dist.tarball %>
</a>
(<%- package.dist.size %>)
</td>
</tr>
<% } %>

View File

@@ -20,7 +20,7 @@
<a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
<a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
</div>
<% } else if (packages[0].name !== keyword) { %>
<% } else if (!match) { %>
<div class="alert alert-info">
Can not found package <%= keyword %>. You can
<a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
@@ -32,10 +32,19 @@
Packages match "<span style="color: #09f;"><%= keyword %></span>"
</h1>
<hr />
<% if (match) { %>
<div class="package match">
<a href="/package/<%= match.name %>" class="package-name"><%= match.name %></a>
<span class="package-description"><%= match.description %></span>
</div>
<% } %>
<% for (var i = 0; i < packages.length; i++) {
var item = packages[i];
if (item.name === keyword) {
continue;
}
%>
<div class="package <%= item.name === keyword ? 'match' : '' %>">
<div class="package">
<a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
<span class="package-description"><%= item.description %></span>
</div>

View File

@@ -21,12 +21,14 @@ var config = require('./config');
var registry = require('./servers/registry');
var web = require('./servers/web');
registry.listen(config.registryPort);
web.listen(config.webPort);
registry.listen(config.registryPort, config.bindingHost);
web.listen(config.webPort, config.bindingHost);
console.log('[%s] [worker:%d] Server started, registry server listen at %d, web listen at %d, cluster: %s',
console.log('[%s] [worker:%d] Server started, registry server listen at %s:%d, web listen at %s%d, cluster: %s',
new Date(), process.pid,
config.registryPort, config.webPort, config.enableCluster);
config.bindingHost, config.registryPort,
config.bindingHost, config.webPort,
config.enableCluster);
graceful({
server: [registry, web],