open-git/cnpmjs.org
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: [BREAKING_CHANGE] support abbreviated meta

Browse Source 
- https://github.com/cnpm/cnpmjs.org/issues/1149
- https://github.com/cnpm/cnpmjs.org/issues/1148
This commit is contained in:
fengmk2
2017-03-24 22:54:27 +08:00
parent 373b535bc5
commit 8cf3344b68
16 changed files with 581 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@@ -8,7 +8,7 @@ jshint:
@node_modules/.bin/jshint .
init-database:
@node test/init_db.js
@NODE_ENV=test node test/init_db.js
init-mysql:
@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'

8
config/index.js
View File

@@ -221,8 +221,16 @@ var config = {
// snyk.io root url
snykUrl: 'https://snyk.io',
// https://github.com/cnpm/cnpmjs.org/issues/1149
// if enable this option, must create module_abbreviated and package_readme table in database
enableAbbreviatedMetadata: false,
};
if (process.env.NODE_ENV === 'test') {
config.enableAbbreviatedMetadata = true;
}
if (process.env.NODE_ENV !== 'test') {
var customConfig;
if (process.env.NODE_ENV === 'development') {

96
controllers/registry/package/list.js
View File

@@ -44,6 +44,11 @@ module.exports = function* list() {
}
}
if (config.enableAbbreviatedMetadata && this.accepts('application/vnd.npm.install-v1+json')) {
yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags);
return;
}
var r = yield [
packageService.listModulesByName(name),
packageService.listStarUserNames(name),
@@ -71,7 +76,7 @@ module.exports = function* list() {
debug('show unpublished %j', unpublishedInfo);
if (unpublishedInfo) {
this.status = 404;
this.body = {
this.jsonp = {
_id: orginalName,
name: orginalName,
time: {
@@ -89,7 +94,7 @@ module.exports = function* list() {
if (rows.length === 0) {
if (!this.allowSync) {
this.status = 404;
this.body = {
this.jsonp = {
error: 'not_found',
reason: 'document not found',
};
@@ -194,3 +199,90 @@ module.exports = function* list() {
debug('show module %s: %s, latest: %s', orginalName, rev, latestMod.version);
this.jsonp = info;
};
function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags) {
var rows = yield packageService.listModuleAbbreviatedsByName(name);
debug('show %s got %d rows, %d tags, modifiedTime: %s', name, rows.length, tags.length, modifiedTime);
if (rows.length === 0) {
// check if unpublished
var unpublishedInfo = yield packageService.getUnpublishedModule(name);
debug('show unpublished %j', unpublishedInfo);
if (unpublishedInfo) {
ctx.status = 404;
ctx.jsonp = {
_id: name,
name: name,
time: {
modified: unpublishedInfo.package.time,
unpublished: unpublishedInfo.package,
},
_attachments: {}
};
return;
}
}
// if module not exist in this registry,
// sync the module backend and return package info from official registry
if (rows.length === 0) {
if (!ctx.allowSync) {
ctx.status = 404;
ctx.jsonp = {
error: 'not_found',
reason: 'document not found',
};
return;
}
// start sync
var logId = yield SyncModuleWorker.sync(name, 'sync-by-install');
debug('start sync %s, get log id %s', name, logId);
return ctx.redirect(config.officialNpmRegistry + this.url);
}
var latestMod = null;
// set tags
var distTags = {};
for (var i = 0; i < tags.length; i++) {
var t = tags[i];
distTags[t.tag] = t.version;
}
// set versions and times
var versions = {};
for (var i = 0; i < rows.length; i++) {
var row = rows[i];
var pkg = row.package;
common.setDownloadURL(pkg, ctx);
pkg._cnpm_publish_time = row.publish_time;
pkg.publish_time = pkg.publish_time || row.publish_time;
versions[pkg.version] = pkg;
if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
latestMod = row;
}
}
if (!latestMod) {
latestMod = rows[0];
}
if (tags.length === 0) {
// some sync error reason, will cause tags missing
// set latest tag at least
distTags.latest = latestMod.package.version;
}
var info = {
name: name,
modified: modifiedTime,
"dist-tags": distTags,
versions: versions,
};
debug('show %j', info);
ctx.jsonp = info;
}

164
controllers/sync_module_worker.js
View File

@@ -295,12 +295,12 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
// try to sync from official replicate when source npm registry is not cnpmjs.org
const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
const versions = yield this.syncByName(concurrencyId, name, registry);
let versions = yield this.syncByName(concurrencyId, name, registry);
if (versions && versions.length === 0 && registry === config.officialNpmReplicate) {
// need to sync sourceNpmRegistry also
// make sure package data be update event replicate down.
// https://github.com/npm/registry/issues/129
yield this.syncByName(concurrencyId, name, config.officialNpmRegistry);
versions = yield this.syncByName(concurrencyId, name, config.officialNpmRegistry);
}
};
@@ -447,14 +447,14 @@ function* _saveNpmUser(username) {
function* _saveMaintainer(modName, username, action) {
if (action === 'add') {
yield* packageService.addPublicModuleMaintainer(modName, username);
yield packageService.addPublicModuleMaintainer(modName, username);
} else if (action === 'remove') {
yield* packageService.removePublicModuleMaintainer(modName, username);
yield packageService.removePublicModuleMaintainer(modName, username);
}
}
SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
var mods = yield* packageService.listModulesByName(name);
var mods = yield packageService.listModulesByName(name);
this.log(' [%s] start unpublished %d versions from local cnpm registry',
name, mods.length);
if (common.isLocalModule(mods)) {
@@ -463,13 +463,13 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
return [];
}
var r = yield* packageService.saveUnpublishedModule(name, unpublishedInfo);
var r = yield packageService.saveUnpublishedModule(name, unpublishedInfo);
this.log(' [%s] save unpublished info: %j to row#%s',
name, unpublishedInfo, r.id);
if (mods.length === 0) {
return;
}
yield [packageService.removeModulesByName(name), packageService.removeModuleTags(name)];
yield [ packageService.removeModulesByName(name), packageService.removeModuleTags(name) ];
var keys = [];
for (var i = 0; i < mods.length; i++) {
var row = mods[i];
@@ -531,6 +531,11 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
localVersionNames.push(r.version);
}
var latestVersionPackageReadme = {
version: pkg['dist-tags'].latest,
readme: pkg.readme,
};
var tags = {};
for (var i = 0; i < tagRows.length; i++) {
var r = tagRows[i];
@@ -541,6 +546,37 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
tags[r.tag] = r.version;
}
// get package AbbreviatedMetadata
var abbreviatedMetadatas = {};
if (config.enableAbbreviatedMetadata) {
var packageUrl = '/' + name.replace('/', '%2f');
var result = yield npmSerivce.request(packageUrl, {
dataType: 'text',
registry: config.sourceNpmRegistry,
headers: {
Accept: 'application/vnd.npm.install-v1+json',
},
});
if (result.status === 200) {
var data;
try {
data = JSON.parse(result.data);
} catch (err) {
that.log(' [%s] get abbreviated meta error: %s, headers: %j, %j',
name, err, result.headers, result.data);
}
if (data) {
var versions = data && data.versions || {};
for (var version in versions) {
const item = versions[version];
if (item && typeof item._hasShrinkwrap === 'boolean') {
abbreviatedMetadatas[version] = { _hasShrinkwrap: item._hasShrinkwrap };
}
}
}
}
}
var missingVersions = [];
var missingTags = [];
var missingDescriptions = [];
@@ -551,6 +587,11 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
// [[user, 'add or remove'], ...]
var diffNpmMaintainers = [];
// [
// { name, version, _hasShrinkwrap }
// ]
var missingAbbreviatedMetadatas = [];
// find out new maintainers
var pkgMaintainers = pkg.maintainers || [];
if (Array.isArray(pkgMaintainers)) {
@@ -624,7 +665,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
if (!version || !version.dist || !version.dist.tarball) {
continue;
}
//patch for readme
// patch for readme
if (!version.readme) {
version.readme = pkg.readme;
}
@@ -633,8 +674,9 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
if (!version.maintainers || !version.maintainers[0]) {
version.maintainers = pkg.maintainers;
}
if (exists.package &&
exists.package.dist.shasum === version.dist.shasum) {
var abbreviatedMetadata = abbreviatedMetadatas[version.version];
if (exists.package && exists.package.dist.shasum === version.dist.shasum) {
// * shasum make sure equal
if ((version.publish_time === exists.publish_time) ||
(!version.publish_time && exists.publish_time)) {
@@ -671,9 +713,28 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
deprecated: undefined,
});
}
// find missing abbreviatedMetadata
if (abbreviatedMetadata) {
for (var key in abbreviatedMetadata) {
if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) {
missingAbbreviatedMetadatas.push(Object.assign({
id: exists.id,
name: exists.package.name,
version: exists.package.version,
}, abbreviatedMetadata));
break;
}
}
}
continue;
}
}
// set abbreviatedMetadata to version package
if (abbreviatedMetadata) {
Object.assign(version, abbreviatedMetadata);
}
missingVersions.push(version);
}
@@ -769,7 +830,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
// sync missing tags
function* syncTag() {
if (deletedTags.length > 0) {
yield* packageService.removeModuleTagsByNames(name, deletedTags);
yield packageService.removeModuleTagsByNames(name, deletedTags);
that.log(' [%s] deleted %d tags: %j',
name, deletedTags.length, deletedTags);
}
@@ -818,6 +879,48 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
}
}
function* syncAbbreviatedMetadatas() {
if (missingAbbreviatedMetadatas.length === 0) {
return;
}
that.log(' [%s] saving %d abbreviated meta datas', name, missingAbbreviatedMetadatas.length);
var res = yield gather(missingAbbreviatedMetadatas.map(function (item) {
return packageService.updateModuleAbbreviatedPackage(item);
}));
for (var i = 0; i < res.length; i++) {
var item = missingAbbreviatedMetadatas[i];
var r = res[i];
if (r.error) {
that.log(' save error, module: %s@%s, error: %s', item.name, item.version, r.error.message);
} else {
that.log(' saved, module_abbreviated: %s@%s, %j', item.name, item.version, item);
}
}
var res = yield gather(missingAbbreviatedMetadatas.map(function (item) {
var fields = {};
for (var key in item) {
if (key === 'id' || key === 'name' || key === 'version') {
continue;
}
fields[key] = item[key];
}
return packageService.updateModulePackageFields(item.id, fields);
}));
for (var i = 0; i < res.length; i++) {
var item = missingAbbreviatedMetadatas[i];
var r = res[i];
if (r.error) {
that.log(' save error, module id: %s, error: %s', item.id, r.error.message);
} else {
that.log(' saved, module id: %s, %j', item.id, item);
}
}
}
function *syncDeprecateds() {
if (missingDeprecateds.length === 0) {
return;
@@ -919,15 +1022,25 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
}
}
yield [
syncDes(),
syncTag(),
syncReadme(),
syncDeprecateds(),
syncMissingStarUsers(),
syncMissingUsers(),
syncNpmPackageMaintainers(),
];
if ( latestVersionPackageReadme.version && latestVersionPackageReadme.readme) {
var existsPackageReadme = yield packageService.getPackageReadme(name);
if (!existsPackageReadme ||
existsPackageReadme.version !== latestVersionPackageReadme.version ||
existsPackageReadme.readme !== latestVersionPackageReadme.readme) {
var r = yield packageService.savePackageReadme(name, latestVersionPackageReadme.readme, latestVersionPackageReadme.version);
that.log(' save packageReadme: %j', r);
}
}
yield syncDes();
yield syncTag();
yield syncReadme();
yield syncDeprecateds();
yield syncMissingStarUsers();
yield syncMissingUsers();
yield syncNpmPackageMaintainers();
yield syncAbbreviatedMetadatas();
return syncedVersionNames;
};
@@ -1110,14 +1223,21 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
mod.package.dist = dist;
var r = yield packageService.saveModule(mod);
var moduleAbbreviatedId = null;
if (config.enableAbbreviatedMetadata) {
var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod);
moduleAbbreviatedId = moduleAbbreviatedResult.id;
}
that.log(' [%s:%s] done, insertId: %s, author: %s, version: %s, '
+ 'size: %d, publish_time: %j, publish on cnpm: %s',
+ 'size: %d, publish_time: %j, publish on cnpm: %s, '
+ 'moduleAbbreviatedId: %s',
sourcePackage.name, versionIndex,
r.id,
author, mod.version, dataSize,
new Date(mod.publish_time),
that._publish);
that._publish,
moduleAbbreviatedId);
return r;
}

28
docs/db.sql
View File

@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS `user` (
`roles` varchar(200) NOT NULL DEFAULT '[]',
`rev` varchar(40) NOT NULL,
`email` varchar(400) NOT NULL,
`json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
`json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'json details',
`npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false',
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`),
@@ -87,6 +87,32 @@ CREATE TABLE IF NOT EXISTS `module` (
-- show create table module\G
-- ALTER TABLE `module` CHANGE `name` `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
CREATE TABLE IF NOT EXISTS `module_abbreviated` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
`version` varchar(30) NOT NULL COMMENT 'module version',
`package` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'the abbreviated metadata',
`publish_time` bigint(20) unsigned,
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`,`version`),
KEY `gmt_modified` (`gmt_modified`),
KEY `publish_time` (`publish_time`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module abbreviated info';
CREATE TABLE IF NOT EXISTS `package_readme` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
`readme` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'the latest version readme',
`version` varchar(30) NOT NULL COMMENT 'module version',
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='package latest readme';
CREATE TABLE IF NOT EXISTS `module_log` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',

14
lib/common.js
View File

@@ -1,19 +1,5 @@
/**!
* cnpmjs.org - lib/common.js
*
* Copyright(c) cnpmjs.org and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var crypto = require('crypto');
var path = require('path');
var config = require('../config');

40
models/index.js
View File

@@ -1,24 +1,16 @@
/**!
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
*/
'use strict';
/**
* Module dependencies.
*/
var path = require('path');
var config = require('../config');
var sequelize = require('../common/sequelize');
function load(name) {
return sequelize.import(path.join(__dirname, name));
}
var _ModuleAbbreviated = config.enableAbbreviatedMetadata ? load('module_abbreviated') : null;
var _PackageReadme = config.enableAbbreviatedMetadata ? load('package_readme') : null;
module.exports = {
sequelize: sequelize,
Module: load('module'),
@@ -44,7 +36,27 @@ module.exports = {
return data[1];
},
queryOne: function* (sql, args) {
var rows = yield* this.query(sql, args);
var rows = yield this.query(sql, args);
return rows && rows[0];
}
},
get ModuleAbbreviated() {
if (!config.enableAbbreviatedMetadata) {
return null;
}
if (!_ModuleAbbreviated) {
_ModuleAbbreviated = load('module_abbreviated');
}
return _ModuleAbbreviated;
},
get PackageReadme() {
if (!config.enableAbbreviatedMetadata) {
return null;
}
if (!_PackageReadme) {
_PackageReadme = load('package_readme');
}
return _PackageReadme;
},
};

14
models/init_script.js
View File

@@ -1,19 +1,5 @@
/**!
* cnpmjs.org - models/init_script.js
*
* Copyright(c) fengmk2 and other contributors.
* MIT Licensed
*
* Authors:
* fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
'use strict';
/**
* Module dependencies.
*/
var config = require('../config');
config.database.logging = console.log;

46
models/module_abbreviated.js Normal file
View File

@@ -0,0 +1,46 @@
'use strict';
module.exports = function (sequelize, DataTypes) {
return sequelize.define('ModuleAbbreviated', {
name: {
type: DataTypes.STRING(100),
allowNull: false,
comment: 'module name'
},
version: {
type: DataTypes.STRING(30),
allowNull: false,
comment: 'module version'
},
package: {
type: DataTypes.LONGTEXT,
comment: 'package.json',
},
publish_time: {
type: DataTypes.BIGINT(20),
allowNull: true,
}
}, {
tableName: 'module_abbreviated',
comment: 'module abbreviated info',
indexes: [
{
unique: true,
fields: ['name', 'version']
},
{
fields: ['gmt_modified']
},
{
fields: ['publish_time']
},
],
classMethods: {
findByNameAndVersion: function* (name, version) {
return yield this.find({
where: { name: name, version: version }
});
}
}
});
};

39
models/package_readme.js Normal file
View File

@@ -0,0 +1,39 @@
'use strict';
module.exports = function (sequelize, DataTypes) {
return sequelize.define('PackageReadme', {
name: {
type: DataTypes.STRING(100),
allowNull: false,
comment: 'module name'
},
version: {
type: DataTypes.STRING(30),
allowNull: false,
comment: 'module latest version'
},
readme: {
type: DataTypes.LONGTEXT,
comment: 'latest version readme',
},
}, {
tableName: 'package_readme',
comment: 'package latest readme',
indexes: [
{
unique: true,
fields: ['name']
},
{
fields: ['gmt_modified']
},
],
classMethods: {
findByName: function* (name) {
return yield this.find({
where: { name: name },
});
}
}
});
};

1
package.json
View File

@@ -6,6 +6,7 @@
"scripts": {
"dev": "DEBUG=cnpm* node dispatch.js",
"test": "make jshint && make test",
"test-local": "make test",
"start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
"status": "./bin/nodejsctl status",
"stop": "./bin/nodejsctl stop"

6
services/npm.js
View File

@@ -11,9 +11,9 @@ function* request(url, options) {
options = options || {};
options.dataType = options.dataType || 'json';
options.timeout = options.timeout || 120000;
options.headers = {
options.headers = Object.assign({
'user-agent': USER_AGENT
};
}, options.headers);
options.gzip = true;
options.followRedirect = true;
var registry = options.registry || config.sourceNpmRegistry;
@@ -52,7 +52,7 @@ exports.getUser = function* (name) {
};
exports.get = function* (name) {
var r = yield* request('/' + name);
var r = yield request('/' + name);
var data = r.data;
if (r.status === 404) {
data = null;

124
services/package.js
View File

@@ -3,6 +3,7 @@
var semver = require('semver');
var models = require('../models');
var common = require('./common');
var config = require('../config');
var Tag = models.Tag;
var User = models.User;
var Module = models.Module;
@@ -49,17 +50,17 @@ exports.getModuleById = function* (id) {
};
exports.getModule = function* (name, version) {
var row = yield* Module.findByNameAndVersion(name, version);
var row = yield Module.findByNameAndVersion(name, version);
parseRow(row);
return row;
};
exports.getModuleByTag = function* (name, tag) {
var tag = yield* Tag.findByNameAndTag(name, tag);
var tag = yield Tag.findByNameAndTag(name, tag);
if (!tag) {
return null;
}
return yield* exports.getModule(tag.name, tag.version);
return yield exports.getModule(tag.name, tag.version);
};
exports.getModuleByRange = function* (name, range) {
@@ -82,7 +83,7 @@ exports.getModuleByRange = function* (name, range) {
};
exports.getLatestModule = function* (name) {
return yield* exports.getModuleByTag(name, 'latest');
return yield exports.getModuleByTag(name, 'latest');
};
// module:list
@@ -195,7 +196,7 @@ exports.listPublicModuleNamesByUser = function* (username) {
});
// find from npm module maintainer table
var moduleNames = yield* NpmModuleMaintainer.listModuleNamesByUser(username);
var moduleNames = yield NpmModuleMaintainer.listModuleNamesByUser(username);
moduleNames.forEach(function (name) {
if (!map[name]) {
names.push(name);
@@ -275,7 +276,7 @@ exports.saveModule = function* (mod) {
// dist.shasum = '';
// dist.size = 0;
var publish_time = mod.publish_time || Date.now();
var item = yield* Module.findByNameAndVersion(mod.name, mod.version);
var item = yield Module.findByNameAndVersion(mod.name, mod.version);
if (!item) {
item = Module.build({
name: mod.name,
@@ -316,12 +317,65 @@ exports.saveModule = function* (mod) {
if (words.length > 0) {
// add keywords
yield* exports.addKeywords(mod.name, description, words);
yield exports.addKeywords(mod.name, description, words);
}
return result;
};
exports.listModuleAbbreviatedsByName = function* (name) {
var rows = yield models.ModuleAbbreviated.findAll({
where: {
name: name
},
order: [ ['id', 'DESC'] ],
});
for (var row of rows) {
row.package = JSON.parse(row.package);
}
return rows;
};
// https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
exports.saveModuleAbbreviated = function* (mod) {
var pkg = JSON.stringify({
name: mod.package.name,
version: mod.package.version,
dependencies: mod.package.dependencies,
optionalDependencies: mod.package.optionalDependencies,
devDependencies: mod.package.devDependencies,
bundleDependencies: mod.package.bundleDependencies,
peerDependencies: mod.package.peerDependencies,
bin: mod.package.bin,
directories: mod.package.directories,
dist: mod.package.dist,
engines: mod.package.engines,
_hasShrinkwrap: mod.package._hasShrinkwrap,
_publish_on_cnpm: mod.package._publish_on_cnpm,
});
var publish_time = mod.publish_time || Date.now();
var item = yield models.ModuleAbbreviated.findByNameAndVersion(mod.name, mod.version);
if (!item) {
item = models.ModuleAbbreviated.build({
name: mod.name,
version: mod.version,
});
}
item.publish_time = publish_time;
item.package = pkg;
if (item.changed()) {
item = yield item.save();
}
var result = {
id: item.id,
gmt_modified: item.gmt_modified,
};
return result;
};
exports.updateModulePackage = function* (id, pkg) {
var mod = yield Module.findById(Number(id));
if (!mod) {
@@ -344,18 +398,31 @@ exports.updateModulePackageFields = function* (id, fields) {
return yield exports.updateModulePackage(id, pkg);
};
exports.updateModuleAbbreviatedPackage = function* (item) {
// item => { id, name, version, _hasShrinkwrap }
var mod = yield models.ModuleAbbreviated.findByNameAndVersion(item.name, item.version);
mod.package = JSON.parse(mod.package);
for (var key in item) {
if (key === 'name' || key === 'version' || key === 'id') {
continue;
}
mod.package[key] = item[key];
}
return yield mod.save([ 'package' ]);
};
exports.updateModuleReadme = function* (id, readme) {
var mod = yield* exports.getModuleById(id);
var mod = yield exports.getModuleById(id);
if (!mod) {
return null;
}
var pkg = mod.package || {};
pkg.readme = readme;
return yield* exports.updateModulePackage(id, pkg);
return yield exports.updateModulePackage(id, pkg);
};
exports.updateModuleDescription = function* (id, description) {
var mod = yield* exports.getModuleById(id);
var mod = yield exports.getModuleById(id);
if (!mod) {
return null;
}
@@ -381,6 +448,39 @@ exports.updateModuleLastModified = function* (name) {
return yield row.save();
};
// try to return latest version readme
exports.getPackageReadme = function* (name) {
if (config.enableAbbreviatedMetadata) {
var row = yield models.PackageReadme.findByName(name);
if (row) {
return {
version: row.version,
readme: row.readme,
};
}
}
var mod = yield exports.getLatestModule(name);
if (mod) {
return {
version: mod.package.version,
readme: mod.package.readme,
};
}
};
exports.savePackageReadme = function* (name, readme, latestVersion) {
var item = yield models.PackageReadme.find({ where: { name: name } });
if (!item) {
item = models.PackageReadme.build({
name: name,
});
}
item.readme = readme;
item.version = latestVersion;
return yield item.save();
};
exports.removeModulesByName = function* (name) {
yield Module.destroy({
where: {
@@ -401,12 +501,12 @@ exports.removeModulesByNameAndVersions = function* (name, versions) {
// tags
exports.addModuleTag = function* (name, tag, version) {
var mod = yield* exports.getModule(name, version);
var mod = yield exports.getModule(name, version);
if (!mod) {
return null;
}
var row = yield* Tag.findByNameAndTag(name, tag);
var row = yield Tag.findByNameAndTag(name, tag);
if (!row) {
row = Tag.build({
name: name,

82
test/controllers/registry/package/list.test.js
View File

@@ -1,6 +1,7 @@
'use strict';
var should = require('should');
var assert = require('assert');
var request = require('supertest');
var mm = require('mm');
var pedding = require('pedding');
@@ -193,4 +194,85 @@ describe('test/controllers/registry/package/list.test.js', () => {
});
});
});
describe('list AbbreviatedMeta', () => {
before(done => {
mm(config, 'sourceNpmRegistry', config.officialNpmRegistry);
mm(config, 'syncModel', 'all');
mm(config, 'enableAbbreviatedMetadata', true);
utils.sync('pedding', done);
});
it('should return abbreviated meta when Accept: application/vnd.npm.install-v1+json', () => {
mm(config, 'syncModel', 'all');
mm(config, 'enableAbbreviatedMetadata', true);
return request(app.listen())
.get('/pedding')
.set('Accept', 'application/vnd.npm.install-v1+json')
.expect(200)
.expect(res => {
const data = res.body;
assert(data.name === 'pedding');
assert(data.modified);
assert(data['dist-tags'].latest);
assert(Object.keys(data.versions).length > 0);
for (const v in data.versions) {
assert('_hasShrinkwrap' in data.versions[v]);
}
});
});
it('should 404 when package not exists', () => {
mm(config, 'syncModel', 'all');
mm(config, 'enableAbbreviatedMetadata', true);
return request(app.listen())
.get('/@cnpmtest/not-exists-package')
.set('Accept', 'application/vnd.npm.install-v1+json')
.expect(404)
.expect({
error: 'not_found',
reason: 'document not found'
});
});
it('should return full meta when enableAbbreviatedMetadata is false and Accept is application/vnd.npm.install-v1+json', () => {
mm(config, 'syncModel', 'all');
mm(config, 'enableAbbreviatedMetadata', false);
return request(app.listen())
.get('/pedding')
.set('Accept', 'application/vnd.npm.install-v1+json')
.expect(200)
.expect(res => {
const data = res.body;
assert(data.name === 'pedding');
assert(data.description);
assert(data.readme);
assert(data['dist-tags'].latest);
assert(Object.keys(data.versions).length > 0);
for (const v in data.versions) {
assert('_hasShrinkwrap' in data.versions[v]);
}
});
});
it('should return full meta when Accept is not application/vnd.npm.install-v1+json', () => {
mm(config, 'syncModel', 'all');
mm(config, 'enableAbbreviatedMetadata', true);
return request(app.listen())
.get('/pedding')
.set('Accept', 'application/json')
.expect(200)
.expect(res => {
const data = res.body;
assert(data.name === 'pedding');
assert(data.description);
assert(data.readme);
assert(data['dist-tags'].latest);
assert(Object.keys(data.versions).length > 0);
for (const v in data.versions) {
assert('_hasShrinkwrap' in data.versions[v]);
}
});
});
});
});

4
test/init.js
View File

@@ -1,9 +1,5 @@
'use strict';
/**
* Module dependencies.
*/
var config = require('../config');
if (process.env.DB) {

4
test/init_db.js
View File

@@ -1,9 +1,5 @@
'use strict';
/**
* Module dependencies.
*/
var crypto = require('crypto');
var path = require('path');
var childProcess = require('child_process');