open-git/cnpmcore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add since for api sync binary (#802)

Browse Source
This commit is contained in:
killa
2025-05-29 08:11:05 +08:00
committed by GitHub
parent b8f2ac5f85
commit 297bd7a745
9 changed files with 189 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.oxlintrc.json
View File

@@ -136,7 +136,14 @@
// jsdoc
"jsdoc/require-returns": "allow",
"jsdoc/require-param": "allow"
"jsdoc/require-param": "allow",
// import
"consistent-type-specifier-style": "allow",
"no-unassigned-import": "allow",
// for-loop
"no-for-loop": "allow"
},
"ignorePatterns": ["index.d.ts"]
}

9
app/common/adapter/binary/ApiBinary.ts
View File

@@ -21,12 +21,17 @@ export class ApiBinary extends AbstractBinary {
async fetch(
dir: string,
binaryName: string
binaryName: string,
lastData?: Record<string, unknown>
): Promise<FetchResult | undefined> {
const apiUrl =
this.config.cnpmcore.syncBinaryFromAPISource ||
`${this.config.cnpmcore.sourceRegistry}/-/binary`;
const url = `${apiUrl}/${binaryName}${dir}`;
let url = `${apiUrl}/${binaryName}${dir}`;
if (lastData && lastData.lastSyncTime) {
url += `?since=${(lastData.lastSyncTime as Date).toISOString()}&limit=100`;
}
const data = await this.requestJSON(url);
if (!Array.isArray(data)) {
this.logger.warn(

17
app/core/service/BinarySyncerService.ts
View File

@@ -59,10 +59,17 @@ export class BinarySyncerService extends AbstractService {
return await this.binaryRepository.findBinary(targetName, parent, name);
}
public async listDirBinaries(binary: Binary) {
public async listDirBinaries(
binary: Binary,
options?: {
limit: number;
since: string;
}
) {
return await this.binaryRepository.listBinaries(
binary.category,
`${binary.parent}${binary.name}`
`${binary.parent}${binary.name}`,
options
);
}
@@ -114,6 +121,12 @@ export class BinarySyncerService extends AbstractService {
lastData[platform] = binaryDir.name.slice(0, -1);
}
}
const latestBinary = await this.binaryRepository.findLatestBinary(
'chromium-browser-snapshots'
);
if (latestBinary) {
lastData.lastSyncTime = latestBinary.date;
}
}
try {
return await this.taskService.createTask(

35
app/port/controller/BinarySyncController.ts
View File

@@ -5,6 +5,7 @@ import {
HTTPMethod,
HTTPMethodEnum,
HTTPParam,
HTTPQuery,
Inject,
Middleware,
HTTPBody,
@@ -60,7 +61,9 @@ export class BinarySyncController extends AbstractController {
async showBinary(
@Context() ctx: EggContext,
@HTTPParam() binaryName: BinaryName,
@HTTPParam() subpath: string
@HTTPParam() subpath: string,
@HTTPQuery() since: string,
@HTTPQuery() limit: string
) {
// check binaryName valid
try {
@@ -68,6 +71,18 @@ export class BinarySyncController extends AbstractController {
} catch {
throw new NotFoundError(`Binary "${binaryName}" not found`);
}
let limitCount: number | undefined;
if (limit) {
limitCount = Number(limit);
if (Number.isNaN(limitCount)) {
throw new NotFoundError(`invalidate limit "${limit}"`);
}
if (limitCount > 1000) {
throw new NotFoundError(
`limit should less than 1000, query is "${limit}"`
);
}
}
subpath = subpath || '/';
if (subpath === '/') {
const items = await this.binarySyncerService.listRootBinaries(binaryName);
@@ -106,7 +121,17 @@ export class BinarySyncController extends AbstractController {
throw new NotFoundError(`Binary "${binaryName}${subpath}" not found`);
}
if (binary.isDir) {
const items = await this.binarySyncerService.listDirBinaries(binary);
let options;
if (limitCount && since) {
options = {
limit: limitCount,
since,
};
}
const items = await this.binarySyncerService.listDirBinaries(
binary,
options
);
return this.formatItems(items);
}
@@ -157,7 +182,9 @@ export class BinarySyncController extends AbstractController {
})
async showBinaryIndex(
@Context() ctx: EggContext,
@HTTPParam() binaryName: BinaryName
@HTTPParam() binaryName: BinaryName,
@HTTPQuery() since: string,
@HTTPQuery() limit: string
) {
// check binaryName valid
try {
@@ -165,7 +192,7 @@ export class BinarySyncController extends AbstractController {
} catch {
throw new NotFoundError(`Binary "${binaryName}" not found`);
}
return await this.showBinary(ctx, binaryName, '/');
return await this.showBinary(ctx, binaryName, '/', since, limit);
}
private formatItems(items: Binary[]) {

27
app/repository/BinaryRepository.ts
View File

@@ -41,9 +41,24 @@ export class BinaryRepository extends AbstractRepository {
async listBinaries(
category: string,
parent: string
parent: string,
options?: {
limit: number;
since: string;
}
): Promise<BinaryEntity[]> {
const models = await this.Binary.find({ category, parent });
let models;
if (options) {
models = await this.Binary.find({
category,
parent,
date: { $gte: options.since },
})
.order('date', 'asc')
.limit(options.limit);
} else {
models = await this.Binary.find({ category, parent });
}
return models.map(model =>
ModelConvertor.convertModelToEntity(model, BinaryEntity)
);
@@ -62,4 +77,12 @@ export class BinaryRepository extends AbstractRepository {
}
return null;
}
async findLatestBinary(category: string): Promise<BinaryEntity | null> {
const model = await this.Binary.findOne({ category }).order('id', 'desc');
if (model) {
return ModelConvertor.convertModelToEntity(model, BinaryEntity);
}
return null;
}
}

3
sql/ddl_mysql.sql
View File

@@ -12,7 +12,8 @@ CREATE TABLE `binaries` (
PRIMARY KEY (`id`),
UNIQUE KEY `uk_binary_id` (`binary_id`),
UNIQUE KEY `uk_category_parent_name` (`category`,`parent`,`name`),
KEY `idx_category_parent_gmt_create` (`category`, `parent`, `gmt_create`)
KEY `idx_category_parent_gmt_create` (`category`, `parent`, `gmt_create`),
KEY `idx_category_parent_date` (`category`, `parent`, `date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8_unicode_ci COMMENT='binary info'
;

1
sql/ddl_postgresql.sql
View File

@@ -14,6 +14,7 @@ CREATE TABLE binaries (
CREATE UNIQUE INDEX binaries_uk_binary_id ON binaries (binary_id);
CREATE UNIQUE INDEX binaries_uk_category_parent_name ON binaries (category, parent, name);
CREATE INDEX binaries_idx_category_parent_gmt_create ON binaries (category, parent, gmt_create);
CREATE INDEX binaries_idx_category_parent_date ON binaries (category, parent, date);
COMMENT ON TABLE binaries IS 'binary info';
COMMENT ON COLUMN binaries.id IS 'primary key';

98
test/port/controller/BinarySyncController/showBinary.test.ts
View File

@@ -757,5 +757,103 @@ describe('test/port/controller/BinarySyncController/showBinary.test.ts', () => {
'https://cdn.mock.com/binaries/node-canvas-prebuilt/v2.6.1/node-canvas-prebuilt-v2.6.1-node-v57-linux-glibc-x64.tar.gz'
);
});
it('since and limit should show valid files', async () => {
await binaryRepository.saveBinary(
Binary.create({
category: 'node-canvas-prebuilt',
parent: '/',
name: 'v2.6.1/',
isDir: true,
size: 0,
date: '2021-12-14T13:12:31.587Z',
})
);
await binaryRepository.saveBinary(
Binary.create({
category: 'node-canvas-prebuilt',
parent: '/v2.6.1/',
name: 'node-canvas-prebuilt-v2.6.1-node-v57-linux-glibc-x64.tar.gz',
isDir: false,
size: 10,
date: '2021-12-15T13:12:31.587Z',
})
);
await binaryRepository.saveBinary(
Binary.create({
category: 'node-canvas-prebuilt',
parent: '/v2.6.1/',
name: 'node-canvas-prebuilt-v2.6.1-node-v58-linux-glibc-x64.tar.gz',
isDir: false,
size: 10,
date: '2021-12-16T13:12:31.587Z',
})
);
await binaryRepository.saveBinary(
Binary.create({
category: 'node-canvas-prebuilt',
parent: '/v2.6.1/',
name: 'node-canvas-prebuilt-v2.6.1-node-v59-linux-glibc-x64.tar.gz',
isDir: false,
size: 10,
date: '2021-12-17T13:12:31.587Z',
})
);
const res = await app
.httpRequest()
.get(
`/-/binary/node-canvas-prebuilt/v2.6.1/?since=2021-12-15T13:12:31.587Z&limit=1`
);
assert.ok(res.status === 200);
assert.ok(
res.headers['content-type'] === 'application/json; charset=utf-8'
);
const items = TestUtil.pickKeys(res.body, [
'category',
'name',
'date',
'type',
'url',
]);
assert.equal(items.length, 1);
assert.deepStrictEqual(items, [
{
category: 'node-canvas-prebuilt',
name: 'node-canvas-prebuilt-v2.6.1-node-v57-linux-glibc-x64.tar.gz',
date: '2021-12-15T13:12:31.587Z',
type: 'file',
url: 'http://localhost:7001/-/binary/node-canvas-prebuilt/v2.6.1/node-canvas-prebuilt-v2.6.1-node-v57-linux-glibc-x64.tar.gz',
},
]);
const res2 = await app
.httpRequest()
.get(
`/-/binary/node-canvas-prebuilt/v2.6.1/?since=2021-12-16T13:12:31.587Z&limit=1`
);
assert.ok(res2.status === 200);
assert.ok(
res2.headers['content-type'] === 'application/json; charset=utf-8'
);
const items2 = TestUtil.pickKeys(res2.body, [
'category',
'name',
'date',
'type',
'url',
]);
assert.equal(items2.length, 1);
assert.deepStrictEqual(items2, [
{
category: 'node-canvas-prebuilt',
name: 'node-canvas-prebuilt-v2.6.1-node-v58-linux-glibc-x64.tar.gz',
date: '2021-12-16T13:12:31.587Z',
type: 'file',
url: 'http://localhost:7001/-/binary/node-canvas-prebuilt/v2.6.1/node-canvas-prebuilt-v2.6.1-node-v58-linux-glibc-x64.tar.gz',
},
]);
});
});
});

4
test/port/controller/HomeController/showTotal.test.ts
View File

@@ -331,7 +331,7 @@ describe('test/port/controller/HomeController/showTotal.test.ts', () => {
.expect('content-type', 'application/json; charset=utf-8');
const data = res.body;
assert.ok(data.upstream_registries.length === 2);
const [defaultRegistry] = data.upstream_registries.filter(
const defaultRegistry = data.upstream_registries.find(
(item: UpstreamRegistryInfo) => item.registry_name === 'default'
);
assert.ok(defaultRegistry.registry_name === 'default');
@@ -343,7 +343,7 @@ describe('test/port/controller/HomeController/showTotal.test.ts', () => {
defaultRegistry.source_registry === 'https://registry.npmjs.org'
);
const [customRegistry] = data.upstream_registries.filter(
const customRegistry = data.upstream_registries.find(
(item: UpstreamRegistryInfo) => item.registry_name === 'custom'
);
assert.ok(customRegistry.registry_name === 'custom');