Release 1.3.1

use simple fs storage in develop env

.npmignore

@@ -19,3 +19,4 @@ coverage/
 .DS_Store
 config/web_readme.md
 .dist/
+config/config.js

History.md

@@ -1,4 +1,22 @@
 
+1.3.1 / 2014-08-18 
+==================
+
+  * add sync not exist tools, close [#424](https://github.com/cnpm/cnpmjs.org/issues/424)
+  * use gittip instand of alipay. close [#425](https://github.com/cnpm/cnpmjs.org/issues/425)
+  * update registry api doc
+
+1.3.0 / 2014-08-11
+==================
+
+  * ignore config/config.js
+  * Merge pull request [#421](https://github.com/cnpm/cnpmjs.org/issues/421) from cnpm/qn-cnpm
+  * fix test case
+  * use fs-cnpm
+  * fix test
+  * use qn-cnpm
+  * bump cfork
+
 1.2.2 / 2014-08-08
 ==================
 
@@ -14,17 +32,17 @@
 
  * show deprecated message
  * Sync deprecated field if it missing
- * Support $ cnpm deprecate [pkgname]@[version] "message". fixed #415
+ * Support $ cnpm deprecate [pkgname]@[version] "message". fixed [#415](https://github.com/cnpm/cnpmjs.org/issues/415)
 
 1.1.0 / 2014-08-07
 ==================
 
- * Add user to maintainers when publish. fixed #395
- * List all npm registry api. close #413
+ * Add user to maintainers when publish. fixed [#395](https://github.com/cnpm/cnpmjs.org/issues/395)
+ * List all npm registry api. close [#413](https://github.com/cnpm/cnpmjs.org/issues/413)
  * limit list since
  * change deps by "~"
  * use cfork to make sure worker fork and restart
- * handle master uncaughtException. fixed #403
+ * handle master uncaughtException. fixed [#403](https://github.com/cnpm/cnpmjs.org/issues/403)
 
 1.0.6 / 2014-08-02
 ==================
@@ -34,12 +52,12 @@
 1.0.5 / 2014-08-02
 ==================
 
- * unpublish pkg@version bug hotfix. fixed #400
+ * unpublish pkg@version bug hotfix. fixed [#400](https://github.com/cnpm/cnpmjs.org/issues/400)
 
 1.0.4 / 2014-08-01
 ==================
 
- * hotfix #399 use not exists
+ * hotfix [#399](https://github.com/cnpm/cnpmjs.org/issues/399) use not exists
 
 1.0.3 / 2014-08-01
 ==================
@@ -54,7 +72,7 @@
 1.0.1 / 2014-08-01
 ==================
 
-  * Merge pull request #398 from cnpm/fix-auth
+  * Merge pull request [#398](https://github.com/cnpm/cnpmjs.org/issues/398) from cnpm/fix-auth
   * hot fix auth error
 
 1.0.0 / 2014-08-01
@@ -85,7 +103,7 @@
  * remove session middleware
  * add DefaultUserService
  * check scopes in module.getAdapt
- * test public mode, fix some logic, close #382
+ * test public mode, fix some logic, close [#382](https://github.com/cnpm/cnpmjs.org/issues/382)
  * move scope.js into publishable.js, add forcePublishWithScope
  * config.scopes not exist, means do not support scope
  * add assert scope middleware
@@ -98,12 +116,12 @@
 0.8.6 / 2014-07-23
 ==================
 
- * show unpublished info on web package page. fixes #381
+ * show unpublished info on web package page. fixes [#381](https://github.com/cnpm/cnpmjs.org/issues/381)
 
 0.8.5 / 2014-07-22
 ==================
 
- * Only private package support default scoped. fixed #378
+ * Only private package support default scoped. fixed [#378](https://github.com/cnpm/cnpmjs.org/issues/378)
 
 0.8.4 / 2014-07-22
 ==================
@@ -124,20 +142,20 @@
 ==================
 
  * add more test cases
- * support default @org. close #376
+ * support default @org. close [#376](https://github.com/cnpm/cnpmjs.org/issues/376)
  * hotfix redis init error
 
 0.8.0 / 2014-07-21
 ==================
 
- * support "scoped" packages. close #352
+ * support "scoped" packages. close [#352](https://github.com/cnpm/cnpmjs.org/issues/352)
  * use safe jsonp
- * Stop support old publish flow. fix #368
+ * Stop support old publish flow. fix [#368](https://github.com/cnpm/cnpmjs.org/issues/368)
  * update SQLs
  * use sync_info and sync_error categories
- * add categories to loggers. fix #370
+ * add categories to loggers. fix [#370](https://github.com/cnpm/cnpmjs.org/issues/370)
  * fix get latest tag always not exists bug
- * support `npm publish --tag beta`. fix #366
+ * support `npm publish --tag beta`. fix [#366](https://github.com/cnpm/cnpmjs.org/issues/366)
  * use mini-logger and error-formater
 
 0.7.0 / 2014-07-07
@@ -145,7 +163,7 @@
 
  * use module_maintainers on GET /pakcage/:name page
  * use new module_maintainers on GET /:name
- * admin user should never publish to other user's packages. fix #363
+ * admin user should never publish to other user's packages. fix [#363](https://github.com/cnpm/cnpmjs.org/issues/363)
  * Add a new table for module-maintainers.
  * gravatar use https
  * support https
@@ -160,8 +178,8 @@
 0.6.0 / 2014-06-16
 ==================
 
- * sync unpublished info. close #353
- * Delete not exists versions on sync worker. #353
+ * sync unpublished info. close [#353](https://github.com/cnpm/cnpmjs.org/issues/353)
+ * Delete not exists versions on sync worker. [#353](https://github.com/cnpm/cnpmjs.org/issues/353)
 
 0.5.3 / 2014-06-13
 ==================

Makefile

@@ -16,6 +16,7 @@ pretest:
 	@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
 	@mysql -uroot 'cnpmjs_test' < ./docs/db.sql
 	@mysql -uroot 'cnpmjs_test' -e 'show tables;'
+	@rm -rf .tmp/dist
 
 test: install pretest
 	@NODE_ENV=test ./node_modules/.bin/mocha \

common/nfs.js

@@ -15,11 +15,5 @@
  */
 
 var config = require('../config');
-var nfs = config.nfs;
 
-if (!nfs) {
-  // use qnfs by default
-  nfs = require('./qnfs');
-}
-
-module.exports = nfs;
+module.exports = config.nfs;

common/qnfs.js

@@ -1,83 +0,0 @@
-/*!
- * cnpmjs.org - common/qnfs.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var qn = require('qn');
-var fs = require('fs');
-var config = require('../config');
-var client = qn.create(config.qn);
-
-thunkify(client, ['delete', 'uploadFile', 'upload', 'download']);
-
-exports._client = client;
-
-/**
- * Upload file
- *
- * @param {String} filepath
- * @param {Object} options
- *  - {String} key
- *  - {Number} size
- */
-exports.upload = function *(filepath, options) {
-  try {
-    yield client.delete(options.key);
-  } catch (err) {
-    // ignore error here
-  }
-
-  var res = yield client.uploadFile(filepath, {
-    key: options.key,
-    size: options.size
-  });
-  var url = res && res[0] ? res[0].url : '';
-  return { url: url };
-};
-
-exports.uploadBuffer = function *(buf, options) {
-  try {
-    yield client.delete(options.key);
-  } catch (err) {
-    // ignore error here
-  }
-
-  var res = yield client.upload(buf, {key: options.key});
-  var url = res && res[0] ? res[0].url : '';
-  return { url: url };
-};
-
-exports.url = function (key) {
-  return client.resourceURL(key);
-};
-
-exports.download = function* (key, filepath, options) {
-  var writeStream = fs.createWriteStream(filepath);
-  yield client.download(key, {
-    timeout: options.timeout,
-    writeStream: writeStream
-  });
-};
-
-exports.remove = function *(key) {
-  try {
-    return yield client.delete(key);
-  } catch (err) {
-    if (err.name === 'QiniuFileNotExistsError') {
-      return;
-    }
-    throw err;
-  }
-};

config/index.js

@@ -128,12 +128,9 @@ var config = {
 
   // package tarball store in qn by default
   // qiniu cdn: http://www.qiniu.com/, it free for dev.
-  qn: {
-    accessKey: "5UyUq-l6jsWqZMU6tuQ85Msehrs3Dr58G-mCZ9rE",
-    secretKey: "YaRsPKiYm4nGUt8mdz2QxeV5Q_yaUzVxagRuWTfM",
-    bucket: "qiniu-sdk-test",
-    domain: "http://qiniu-sdk-test.qiniudn.com",
-  },
+  nfs: require('fs-cnpm')({
+    dir: path.join(root, '.tmp', 'dist')
+  }),
 
   // registry url name
   registryHost: 'r.cnpmjs.org',

controllers/total.js

@@ -32,7 +32,7 @@ exports.show = function *() {
   total.instance_start_time = startTime;
   total.node_version = process.version;
   total.app_version = version;
-  total.donate = 'https://me.alipay.com/imk2';
+  total.donate = 'https://www.gittip.com/fengmk2';
   total.sync_model = config.syncModel;
 
   this.body = total;

docs/registry-api.md

@@ -245,7 +245,9 @@ Content-Length: 2243
 }
 ```
 
-### Get a special version or tag package
+### ~~Get a special version or tag package~~
+
+__deprecated__
 
 ```
 GET /:package/:tag_or_version
@@ -505,6 +507,7 @@ Status: 201 Created
 ### Remove one version from package
 
 * Authentication required.
+* In any delete, note that __the version number still cannot be reused__.
 
 ```
 PUT /:package/-rev/:rev
@@ -512,6 +515,8 @@ PUT /:package/-rev/:rev
 
 #### Input
 
+Remove that specific version from the versions hash in the `PUT` body.
+
 Example for removing `0.0.1` version:
 
 ```json
@@ -551,6 +556,25 @@ Status: 201 Created
 }
 ```
 
+### Remove all versions of a package
+
+* Authentication required.
+* In any delete, note that __the version number still cannot be reused__.
+
+```
+DELETE /:package/-rev/:rev
+```
+
+#### Response
+
+```json
+Status: 201 Created
+
+{
+  "ok": true
+}
+```
+
 ### Remove a tgz file from package
 
 * Authentication required.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "1.2.2",
+  "version": "1.3.1",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "bytes": "~1.0.0",
-    "cfork": "~1.0.1",
+    "cfork": "~1.1.0",
     "cheerio": "~0.17.0",
     "co": "~3.1.0",
     "co-defer": "~0.1.0",
@@ -24,6 +24,7 @@
     "debug": "~1.0.4",
     "error-formater": "~1.0.3",
     "eventproxy": "~0.3.1",
+    "fs-cnpm": "~1.0.2",
     "giturl": "~0.0.3",
     "graceful": "~0.1.0",
     "gravatar": "~1.0.6",
@@ -41,7 +42,6 @@
     "multiline": "~0.3.4",
     "mysql": "~2.4.2",
     "nodemailer": "0.7.1",
-    "qn": "~0.2.2",
     "ready": "~0.1.1",
     "redis": "~0.11.0",
     "semver": "~3.0.1",

test/controllers/registry/module.test.js

@@ -779,11 +779,10 @@ describe('controllers/registry/module.test.js', function () {
   });
 
   describe('GET /:name/download/:filename', function () {
-    it('should download a file with 302 redirect', function (done) {
+    it('should download a file with 200', function (done) {
       request(app)
-      .get('/cutter/download/cutter-0.0.2.tgz')
-      .expect('Location', config.qn.domain + '/cutter/-/cutter-0.0.2.tgz')
-      .expect(302, done);
+      .get('/mk2testmodule/download/mk2testmodule-0.0.1.tgz')
+      .expect(200, done);
     });
   });
 

test/controllers/registry/module/scope_package.test.js

@@ -145,8 +145,7 @@ describe('controllers/registry/module/scope_package.test.js', function () {
   it('should download work', function (done) {
     request(app)
     .get('/@cnpm/test-scope-package/download/@cnpm/test-scope-package-0.0.2.tgz')
-    .expect('Location', /\.tgz$/)
-    .expect(302, done);
+    .expect(200, done);
   });
 
   describe('support adaptScope', function () {

test/controllers/web/dist.test.js

@@ -18,6 +18,8 @@ var should = require('should');
 var request = require('supertest');
 var pedding = require('pedding');
 var mm = require('mm');
+var fs = require('fs');
+var nfs = require('../../../common/nfs');
 var app = require('../../../servers/web');
 var Dist = require('../../../proxy/dist');
 
@@ -127,7 +129,7 @@ describe('controllers/web/dist.test.js', function () {
           url: '/dist/v0.10.28/SHASUMS.txt'
         };
       });
-
+      fs.writeFileSync(nfs._getpath('/dist/v0.10.28/SHASUMS.txt'), '6eff580cc8460741155d42ef1ef537961194443f');
       request(app)
       .get('/dist/v0.10.28/SHASUMS.txt')
       .expect(200)

test/proxy/dist.test.js

@@ -16,34 +16,40 @@
 
 var should = require('should');
 var Dist = require('../../proxy/dist');
+var fs = require('fs');
+var nfs = require('../../common/nfs');
 
 describe('proxy/dist.test.js', function () {
   describe('savefile() and getfile', function () {
     it('should save and get /npm-versions.txt', function* () {
+      var name = 'npm-versions.txt';
       var info = {
-        name: 'npm-versions.txt',
+        name: name,
         parent: '/',
         date: '15-Sep-2011 23:48',
         size: 1676,
-        url: 'http://cnpmjs.org/dist/npm-versions.txt',
+        url: name,
         sha1: '104731881047318810473188'
       };
       yield* Dist.savefile(info);
+      fs.writeFileSync(nfs._getpath('npm-versions.txt'), 'npm version');
       var got = yield* Dist.getfile('/npm-versions.txt');
       should.exist(got);
       got.should.eql(info);
     });
 
     it('should save and get /v1.0.0/npm-versions.txt', function* () {
+      var name = 'v1.0.0/npm-versions.txt';
       var info = {
         name: 'npm-versions.txt',
         parent: '/v1.0.0/',
         date: '15-Sep-2011 23:48',
         size: 1676,
-        url: 'http://cnpmjs.org/dist/v1.0.0/npm-versions.txt',
+        url: 'v1.0.0/npm-versions.txt',
         sha1: '104731881047318810473188'
       };
       yield* Dist.savefile(info);
+      fs.writeFileSync(nfs._getpath('npm-versions.txt'), 'npm version 1.0.0');
       var got = yield* Dist.getfile('/v1.0.0/npm-versions.txt');
       should.exist(got);
       got.should.eql(info);

tools/sync_not_exist.js

@@ -0,0 +1,80 @@
+/*!
+ * cnpmjs.org - tools/sync_not_exist.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:tools:sync_not_exist');
+var SyncModuleWorker = require('../proxy/sync_module_worker');
+var thunkify = require('thunkify-wrap');
+var Module = require('../proxy/module');
+var config = require('../config');
+var Npm = require('../proxy/npm');
+var utility = require('utility');
+var co = require('co');
+
+function subtraction(arrOne, arrTwo) {
+  arrOne = arrOne || [];
+  arrTwo = arrTwo || [];
+  var map = {};
+  var results = [];
+  arrTwo.forEach(function (name) {
+    map[name] = true;
+  });
+  arrOne.forEach(function (name) {
+    !map[name] && results.push(name);
+  });
+  return results;
+}
+
+function *sync() {
+  var syncTime = Date.now();
+
+  var allExists = yield Module.listShort();
+  var existPackages = allExists.map(function (p) {
+    return p.name;
+  });
+
+  var allPackages = yield Npm.getShort();
+  var packages = subtraction(allPackages, existPackages);
+  if (!packages.length) {
+    debug('no packages need be sync');
+    return;
+  }
+  debug('Total %d packages to sync', packages.length);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    concurrency: config.syncConcurrency
+  });
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  debug('All packages sync done, successes %d, fails %d',
+    worker.successes.length, worker.fails.length);
+
+  Total.setLastExistSyncTime(syncTime, utility.noop);
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
+}
+
+module.exports = sync;
+
+if (!module.parent) {
+  console.log('[tools/sync_not_exist.js] start sync not exist modules');
+  co(sync)();
+}