open-git/cnpmcore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: revalidate latest version (#573)

Browse Source 
> closes #376: fix issue with incomplete binary file after upstream
release or sync failure
* 🧶 Modify the binary `diff` method, adding latest version check method.
* ♻️ Perform additional comparison on the latest version, without
modifying existing data.

---------
> 修复 #376 ，兼容上游发布或同步失败后，产物同步不全的问题
* 🧶 调整 binary `diff` 方法，添加最新版本校验逻辑
* ♻️ 对最新版本的子目录进行额外比对，存量数据不做修改
This commit is contained in:
elrrrrrrr
2023-08-22 15:41:04 +08:00
committed by GitHub
parent 9916bd9ecf
commit 73b4383f5c
2 changed files with 142 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
app/core/service/BinarySyncerService.ts
View File

@@ -21,6 +21,7 @@ import { AbstractBinary, BinaryItem } from '../../common/adapter/binary/Abstract
import { AbstractService } from '../../common/AbstractService';
import { TaskRepository } from '../../repository/TaskRepository';
import { BinaryType } from '../../common/enum/Binary';
import { sortBy } from 'lodash';
function isoNow() {
return new Date().toISOString();
@@ -152,7 +153,7 @@ export class BinarySyncerService extends AbstractService {
}
}
private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '') {
private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '', latestVersionParent = '/') {
const binaryName = task.targetName as BinaryName;
const result = await binaryAdapter.fetch(dir, binaryName);
let hasDownloadError = false;
@@ -160,14 +161,15 @@ export class BinarySyncerService extends AbstractService {
if (result && result.items.length > 0) {
hasItems = true;
let logs: string[] = [];
const newItems = await this.diff(binaryName, dir, result.items);
const { newItems, latestVersionDir } = await this.diff(binaryName, dir, result.items, latestVersionParent);
logs.push(`[${isoNow()}][${dir}] 🚧 Syncing diff: ${result.items.length} => ${newItems.length}, Binary class: ${binaryAdapter.constructor.name}`);
// re-check latest version
for (const [ index, { item, reason }] of newItems.entries()) {
if (item.isDir) {
logs.push(`[${isoNow()}][${dir}] 🚧 [${parentIndex}${index}] Start sync dir ${JSON.stringify(item)}, reason: ${reason}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`);
const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`, latestVersionDir);
if (hasError) {
hasDownloadError = true;
} else {
@@ -231,7 +233,12 @@ export class BinarySyncerService extends AbstractService {
return [ hasDownloadError, hasItems ];
}
private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[]) {
// see https://github.com/cnpm/cnpmcore/issues/556
// 上游可能正在发布新版本、同步流程中断，导致同步的时候，文件列表不一致
// 如果的当前目录命中 latestVersionParent 父目录，那么就再校验一下当前目录
// 如果 existsItems 为空或者经过修改，那么就不需要 revalidate 了
private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[], latestVersionParent = '/') {
const existsItems = await this.binaryRepository.listBinaries(binaryName, dir);
const existsMap = new Map<string, Binary>();
for (const item of existsItems) {
@@ -262,9 +269,23 @@ export class BinarySyncerService extends AbstractService {
existsItem.sourceUrl = item.url;
existsItem.ignoreDownloadStatuses = item.ignoreDownloadStatuses;
existsItem.date = item.date;
} else if (dir.endsWith(latestVersionParent)) {
const isLatestItem = sortBy(fetchItems, [ 'date' ]).pop()?.name === item.name;
if (isLatestItem && existsItem.isDir) {
diffItems.push({
item: existsItem,
reason: `revalidate latest version, latest parent dir is ${latestVersionParent}, current dir is ${dir}, current name is ${existsItem.name}`,
});
latestVersionParent = `${latestVersionParent}${existsItem.name}`;
}
}
}
return diffItems;
return {
newItems: diffItems,
latestVersionDir: latestVersionParent,
};
}
private async saveBinaryItem(binary: Binary, tmpfile?: string) {

118
test/core/service/BinarySyncerService/executeTask.test.ts
View File

@@ -6,6 +6,7 @@ import { Task as TaskModel } from '../../../../app/repository/model/Task';
import { HistoryTask as HistoryTaskModel } from '../../../../app/repository/model/HistoryTask';
import { NodeBinary } from '../../../../app/common/adapter/binary/NodeBinary';
import { ApiBinary } from '../../../../app/common/adapter/binary/ApiBinary';
import { BinaryRepository } from '../../../../app/repository/BinaryRepository';
describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => {
let binarySyncerService: BinarySyncerService;
@@ -74,7 +75,8 @@ describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => {
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('Syncing diff: 2 => 0'));
assert(log.includes('reason: revalidate latest version'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));
// mock date change
@@ -271,9 +273,121 @@ describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => {
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('Syncing diff: 2 => 0'));
assert(log.includes('reason: revalidate latest version'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));
app.mockAgent().assertNoPendingInterceptors();
});
it('should revalidate latest version', async () => {
app.mockHttpclient('https://nodejs.org/dist/index.json', 'GET', {
data: await TestUtil.readFixturesFile('nodejs.org/site/index.json'),
persist: false,
});
app.mockHttpclient('https://nodejs.org/dist/latest/docs/apilinks.json', 'GET', {
data: await TestUtil.readFixturesFile('nodejs.org/site/latest/docs/apilinks.json'),
persist: false,
});
await binarySyncerService.createTask('node', {});
let task = await binarySyncerService.findExecuteTask();
assert(task);
mock(NodeBinary.prototype, 'fetch', async (dir: string) => {
if (dir === '/') {
return {
items: [
{ name: 'latest/', isDir: true, url: '', size: '-', date: '17-Dec-2021 23:17' },
{ name: 'index.json', isDir: false, url: 'https://nodejs.org/dist/index.json', size: '219862', date: '17-Dec-2021 23:16' },
],
};
}
if (dir === '/latest/') {
return {
items: [
{ name: 'docs/', isDir: true, url: '', size: '-', date: '17-Dec-2021 21:31' },
],
};
}
if (dir === '/latest/docs/') {
return {
items: [
{ name: 'apilinks.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '17-Dec-2021 21:29' },
],
};
}
return { items: [] };
});
await binarySyncerService.executeTask(task);
app.mockAgent().assertNoPendingInterceptors();
assert(!await TaskModel.findOne({ taskId: task.taskId }));
assert(await HistoryTaskModel.findOne({ taskId: task.taskId }));
let stream = await binarySyncerService.findTaskLog(task);
assert(stream);
let log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('Syncing diff: 2 => 2'));
assert(log.includes('[/] 🟢 Synced dir success'));
assert(log.includes('[/latest/] 🟢 Synced dir success'));
assert(log.includes('[/latest/docs/] 🟢 Synced dir success'));
// sync again
await binarySyncerService.createTask('node', {});
task = await binarySyncerService.findExecuteTask();
assert(task);
await binarySyncerService.executeTask(task);
stream = await binarySyncerService.findTaskLog(task);
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('reason: revalidate latest version'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));
// mock version change
// console.log(binaryRepository.findBinary('node'));
// mock upstream updated
mock(NodeBinary.prototype, 'fetch', async (dir: string) => {
if (dir === '/') {
return {
items: [
{ name: 'latest/', isDir: true, url: '', size: '-', date: '17-Dec-2021 23:17' },
{ name: 'index.json', isDir: false, url: 'https://nodejs.org/dist/index.json', size: '219862', date: '17-Dec-2021 23:16' },
],
};
}
if (dir === '/latest/') {
return {
items: [
{ name: 'docs/', isDir: true, url: '', size: '-', date: '17-Dec-2021 21:31' },
],
};
}
if (dir === '/latest/docs/') {
return {
items: [
{ name: 'apilinks.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '17-Dec-2021 21:29' },
{ name: 'apilinks2.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '18-Dec-2021 21:29' },
],
};
}
return { items: [] };
});
await binarySyncerService.createTask('node', {});
task = await binarySyncerService.findExecuteTask();
await binarySyncerService.executeTask(task!);
stream = await binarySyncerService.findTaskLog(task!);
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('"name":"apilinks2.json"'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));
app.mockAgent().assertNoPendingInterceptors();
const binaryRepository = await app.getEggObject(BinaryRepository);
const BinaryItems = await binaryRepository.listBinaries('node', '/latest/docs/');
assert(BinaryItems.length === 2);
});
});
});