Release 0.1.3

make sure all packages name are lower case

.gitignore

@@ -18,3 +18,4 @@ public/dist/
 config/config.js
 backup/*.json
 backup/*.gz
+docs/web/history.md

History.md

@@ -1,5 +1,19 @@
 
-0.1.2 / 2013-12-19 
+0.1.3 / 2013-12-20
+==================
+
+  * add favicon
+  * Fix auth middleware bug (@alsotang)
+  * make sure all packages name are lower case
+  * select ids from tag
+  * fix nodejsctl
+  * fix #112 missing versions and time no sync
+  * remove restart command
+  * fix sync missing packages error
+  * fix web/readme.md, add install
+  * fix #109 pkg no times and no versions bug.
+
+0.1.2 / 2013-12-19
 ==================
 
   * fix times not exists canot sync bug. fixed #101

README.md

@@ -34,12 +34,13 @@ $ git summary
 
  project  : cnpmjs.org
  repo age : 2 weeks
- active   : 73 days
- commits  : 186
+ active   : 81 days
+ commits  : 205
  files    : 83
  authors  :
-   109  fengmk2                 58.6%
-    77  dead_horse              41.4%
+   120  fengmk2                 58.5%
+    84  dead_horse              41.0%
+     1  Alsotang                0.5%
 ```
 
 ## License

bin/nodejsctl

@@ -10,7 +10,8 @@ cd `dirname $0`/..
 NODEJS=node
 BASE_HOME=`pwd`
 PROJECT_NAME=`basename ${BASE_HOME}`
-STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'))"`
+STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\
+process.exit(0);"`
 NODE_DISPATH_PATH=${BASE_HOME}/dispatch.js
 PROG_NAME=$0
 ACTION=$1
@@ -47,9 +48,9 @@ stop()
 {
   get_pid
   if [ ! -z "$PID" ]; then
-    echo "Waiting $PROJECT_NAME stop for 5s ..."
+    echo "Waiting $PROJECT_NAME stop for 2s ..."
     kill -15 $PID
-    sleep 5
+    sleep 2
 
     node_num=`ps -ef |grep node|grep ${PROJECT_NAME}|grep -v grep|wc -l`
     if [ $node_num != 0 ]; then

config/index.js

@@ -68,7 +68,7 @@ var config = {
     user: 'address@gmail.com',
     pass: 'your password',
     ssl: true,
-    debug: true
+    debug: false 
   },
 
   sourceNpmRegistry: 'http://registry.npmjs.org',

docs/web/readme.md

@@ -1,6 +1,6 @@
 # cnpmjs.org: Private npm registry and web for Enterprise
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org)[![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org) [![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
@@ -135,9 +135,17 @@ Or you can just use our `cnpm` cli:
 $ npm install cnpm -g
 ```
 
+### install
+
+Install package from [registry.cnpmjs.org](http://registry.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+
+```
+$ cnpm install [name]
+```
+
 ### sync
 
-Only `cnpm` cli has this command:
+Only `cnpm` cli has this command. Meaning sync package from source npm.
 
 ```bash
 $ cnpm sync connect
@@ -149,33 +157,38 @@ sync package on web: [cnpmjs.org/sync/connect](http://cnpmjs.org/sync/connect)
 $ open http://cnpmjs.org/sync/connect
 ```
 
-### publish
+### publish / unpublish
 
-Meaning sync package from source npm.
-
-Only `admin` user can publish package to private registry.
+Only `admin` user can publish / unpublish package to private registry.
 
 ```bash
 $ cnpm publish [name]
+$ cnpm unpublish [name]
 ```
 
+### Orther command
+Support all the other npm command.
+
 ## TODO list
 
 @see Github [Issues](https://github.com/fengmk2/cnpmjs.org/issues)
 
 ## Authors
 
+Release [History](/history).
+
 ```bash
 $ git summary
 
  project  : cnpmjs.org
  repo age : 2 weeks
- active   : 73 days
- commits  : 186
+ active   : 81 days
+ commits  : 205
  files    : 83
  authors  :
-   109  fengmk2                 58.6%
-    77  dead_horse              41.4%
+   120  fengmk2                 58.5%
+    84  dead_horse              41.0%
+     1  Alsotang                0.5%
 ```
 
 ## npm and cnpm relation

middleware/auth.js

@@ -52,7 +52,7 @@ module.exports = function (options) {
       }
 
       req.session.name = row.name;
-      if (config.admins[req.session.name]) {
+      if (config.admins.hasOwnProperty(req.session.name)) {
         req.session.isAdmin = true;
       }
       debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',

package.json

@@ -1,14 +1,13 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Private npm registry and web",
   "main": "dispatch.js",
   "scripts": {
     "test": "make test-all",
-    "start": "./bin/nodejsctl start && npm run status",
+    "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
-    "stop": "./bin/nodejsctl stop && npm run status",
-    "restart": "./bin/nodejsctl restart && npm run status",
+    "stop": "./bin/nodejsctl stop",
     "blanket": {
       "pattern": "//^((?!(node_modules|test|common)).)*$/",
       "data-cover-flags": {
@@ -20,23 +19,24 @@
     }
   },
   "dependencies": {
-    "humanize-number": "0.0.2",
+    "forward": ">=0.0.4",
+    "humanize-number": ">=0.0.2",
     "gravatar": "1.0.6",
-    "urllib": "0.5.5",
+    "urllib": ">=0.5.5",
     "connect-markdown": "0.0.3",
-    "qn": "0.2.0",
+    "qn": ">=0.2.0",
     "microtime": "0.5.1",
-    "debug": "0.7.4",
-    "utility": "0.1.9",
+    "debug": ">=0.7.4",
+    "utility": ">=0.1.9",
     "ready": "0.1.1",
     "connect": "2.12.0",
     "connect-rt": "0.0.2",
     "connect-redis": "1.4.6",
     "connect-render": "0.3.2",
-    "urlrouter": "0.5.4",
-    "graceful": "0.0.5",
+    "urlrouter": ">=0.5.4",
+    "graceful": ">=0.0.5",
     "moment": "2.4.0",
-    "logfilestream": "0.1.0",
+    "logfilestream": ">=0.1.0",
     "ms": "0.6.2",
     "mkdirp": "0.3.5",
     "mysql": "2.0.0-rc2",
@@ -46,7 +46,7 @@
     "semver": "2.2.1",
     "marked": "0.2.10",
     "nodemailer": "0.5.15",
-    "eventproxy": "0.2.6"
+    "eventproxy": ">=0.2.6"
   },
   "devDependencies": {
     "supertest": "*",

proxy/module.js

@@ -21,9 +21,6 @@ var mysql = require('../common/mysql');
 
 var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, version, description, package, dist_tarball, dist_shasum, dist_size';
 
-// var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, author, name, version, package, dist_tarball, dist_shasum, dist_size) \
-//   VALUES(now(), now(), ?, ?, ?, ?, ?, ?, ?);';
-
 var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, \
   publish_time, author, name, version, package, dist_tarball, dist_shasum, dist_size, description) \
   VALUES(now(), now(), ?, ?, ?, ?, ?, ?, ?, ?, ?) \
@@ -227,35 +224,23 @@ exports.listByName = function (name, callback) {
 };
 
 var LIST_SINCE_SQLS = [
-  'SELECT distinct(name) AS name FROM module WHERE publish_time > ?;',
-  'SELECT module_id FROM tag WHERE tag="latest" AND name IN (?);',
+  'SELECT module_id FROM tag WHERE tag="latest" AND gmt_modified>?',
   'SELECT name, package FROM module WHERE id IN (?);'
 ];
 exports.listSince = function (start, callback) {
   var ep = eventproxy.create();
   ep.fail(callback);
-  mysql.query(LIST_SINCE_SQLS[0], [start], ep.done(function (rows) {
+  mysql.query(LIST_SINCE_SQLS[0], [new Date(start)], ep.done(function (rows) {
     if (!rows || rows.length === 0) {
       return callback(null, []);
     }
-    ep.emit('names', rows.map(function (r) {
-      return r.name;
+    ep.emit('ids', rows.map(function (r) {
+      return r.module_id;
     }));
   }));
 
-  ep.once('names', function (names) {
-    mysql.query(LIST_SINCE_SQLS[1], [names], ep.done(function (rows) {
-      if (!rows || rows.length === 0) {
-        return callback(null, []);
-      }      
-      ep.emit('ids', rows.map(function (r) {
-        return r.module_id;
-      }));
-    }));
-  });
-
   ep.once('ids', function (ids) {
-    mysql.query(LIST_SINCE_SQLS[2], [ids], callback);
+    mysql.query(LIST_SINCE_SQLS[1], [ids], callback);
   });
 };
 
@@ -264,6 +249,11 @@ exports.listShort = function (callback) {
   mysql.query(LIST_SHORT_SQL, callback);
 };
 
+var LIST_ALL_MODULE_NAMES_SQL = 'SELECT distinct(name) FROM module ORDER BY name';
+exports.listAllModuleNames = function (callback) {
+  mysql.query(LIST_ALL_MODULE_NAMES_SQL, callback);
+};
+
 var DELETE_MODULE_BY_NAME_SQL = 'DELETE FROM module WHERE name=?;';
 exports.removeByName = function (name, callback) {
   mysql.query(DELETE_MODULE_BY_NAME_SQL, [name], callback);
@@ -294,7 +284,7 @@ exports.listByAuthor = function (author, callback) {
     mysql.query(LIST_BY_AUTH_SQLS[1], [names], ep.done(function (rows) {
       if (!rows || rows.length === 0) {
         return callback(null, []);
-      }      
+      }
       ep.emit('ids', rows.map(function (r) {
         return r.module_id;
       }));

proxy/sync_module_worker.js

@@ -40,6 +40,11 @@ function SyncModuleWorker(options) {
   }
 
   this.names = options.name;
+  for (var i = 0; i < this.names.length; i++) {
+    // ensure package name is lower case
+    this.names[i] = this.names[i].toLowerCase();
+  }
+
   this.username = options.username;
   this.concurrency = options.concurrency || 1;
   this.syncingNames = {};
@@ -136,7 +141,9 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
   var ep = eventproxy.create();
   ep.fail(callback);
 
+  var hasModules = false;
   Module.listByName(name, ep.done(function (rows) {
+    hasModules = rows.length > 0;
     var map = {};
     for (var i = 0; i < rows.length; i++) {
       var r = rows[i];
@@ -182,10 +189,41 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
   var missingDescriptions = [];
   ep.all('existsMap', 'existsTags', function (map, tags) {
     var times = pkg.time || {};
+    pkg.versions = pkg.versions || {};
     var versionNames = Object.keys(times);
     if (versionNames.length === 0) {
       versionNames = Object.keys(pkg.versions);
     }
+    if (versionNames.length === 0) {
+      that.log('  [%s] no times and no versions, hasModules: %s', pkg.name, hasModules);
+      if (!hasModules) {
+        // save a next module
+        var maintainer = pkg.maintainers && pkg.maintainers[0];
+        if (maintainer && maintainer.name) {
+          maintainer = maintainer.name;
+        }
+        if (!maintainer) {
+          maintainer = '-';
+        }
+        var nextMod = {
+          name: pkg.name,
+          version: 'next',
+          author: maintainer,
+          package: {
+            name: pkg.name,
+            version: 'next',
+            description: pkg.description || '',
+            readme: pkg.readme || '',
+            maintainers: pkg.maintainers || {
+              name: maintainer
+            },
+          },
+        };
+        Module.add(nextMod, function (err, result) {
+          that.log('  [%s] save next module, %j, error: %s', pkg.name, result, err);
+        });
+      }
+    }
     var versions = [];
     for (var i = 0; i < versionNames.length; i++) {
       var v = versionNames[i];

servers/registry.js

@@ -21,12 +21,15 @@ var connect = require('connect');
 var rt = require('connect-rt');
 var responseCookie = require('response-cookie');
 var urlrouter = require('urlrouter');
+var forward = require('forward');
+var path = require('path');
 var routes = require('../routes/registry');
 var logger = require('../common/logger');
 var config = require('../config');
 var session = require('../common/session');
 var auth = require('../middleware/auth');
 
+var rootdir = path.dirname(__dirname);
 var app = connect();
 
 app.use(rt({headerName: 'X-ReadTime'}));
@@ -35,6 +38,8 @@ app.use(function (req, res, next) {
   next();
 });
 
+app.use('/favicon.ico', forward(path.join(rootdir, 'public', 'favicon.png')));
+
 app.use('/dist', connect.static(config.uploadDir));
 
 app.use(responseCookie());

sync/sync_all.js

@@ -14,16 +14,16 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:sync:sync_all');
+var eventproxy = require('eventproxy');
+var ms = require('ms');
+var utility = require('utility');
 var config = require('../config');
+var Status = require('./status');
 var Npm = require('../proxy/npm');
 var Total = require('../proxy/total');
-var eventproxy = require('eventproxy');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
-var debug = require('debug')('cnpmjs.org:sync:sync_all');
-var utility = require('utility');
-var Status = require('./status');
 var Module = require('../proxy/module');
-var ms = require('ms');
 
 function subtract(subtracter, minuend) {
   subtracter = subtracter || [];
@@ -31,10 +31,14 @@ function subtract(subtracter, minuend) {
   var map = {};
   var results = [];
   minuend.forEach(function (name) {
-    map[name] = true;
+    map[name.toLowerCase()] = true;
   });
   subtracter.forEach(function (name) {
-    !map[name] && results.push(name);
+    var lowerName = name.toLowerCase();
+    if (!map[lowerName] && !/[A-Z]/.test(name)) {
+      // ensure package name is lower case
+      results.push(lowerName);
+    }
   });
   return results;
 }
@@ -52,7 +56,7 @@ function union(arrOne, arrTwo) {
 /**
  * when sync from official at the first time
  * get all packages by short and restart from last synced module
- * @param {String} lastSyncModule 
+ * @param {String} lastSyncModule
  * @param {Function} callback
  */
 function getFirstSyncPackages(lastSyncModule, callback) {
@@ -71,8 +75,8 @@ function getFirstSyncPackages(lastSyncModule, callback) {
 
 /**
  * get all the packages that update time > lastSyncTime
- * @param {Number} lastSyncTime 
- * @param {Function} callback 
+ * @param {Number} lastSyncTime
+ * @param {Function} callback
  */
 function getCommonSyncPackages(lastSyncTime, callback) {
   Npm.getAllSince(lastSyncTime, function (err, data) {
@@ -92,7 +96,7 @@ function getMissPackages(callback) {
   var ep = eventproxy.create();
   ep.fail(callback);
   Npm.getShort(ep.doneLater('allPackages'));
-  Module.listShort(ep.doneLater(function (rows) {
+  Module.listAllModuleNames(ep.doneLater(function (rows) {
     var existPackages = rows.map(function (row) {
       return row.name;
     });

test/sync/sync_all.test.js

@@ -47,7 +47,7 @@ describe('sync/sync_all.js', function () {
       });
       mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
-      mm.data(Module, 'listShort', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
+      mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
       sync(function (err, data) {
         should.not.exist(err);
         data.successes.should.eql(['cnpm', 'cnpmjs.org', 'cutter']);