refactor: doc_count & doc_version_count perf (#797)

<!--- SUMMARY_MARKER --->
## Sweep Summary <sub><a href="https://app.sweep.dev"><img
src="https://raw.githubusercontent.com/sweepai/sweep/main/.assets/sweep-square.png"
width="25" alt="Sweep"></a></sub>

Improves database performance by replacing expensive count queries with
a dedicated totals table that's updated asynchronously via events.

- Created a new `totals` table in `app/repository/model/Total.ts` to
store package and version counts instead of running expensive SQL count
queries.
- Implemented `TotalRepository` in `app/repository/TotalRepository.ts`
with methods to increment and retrieve count values.
- Added event handlers in `app/core/event/TotalHandler.ts` that listen
for package and version additions to update counts asynchronously.
- Modified `PackageRepository.queryTotal()` to fetch counts from the
totals table instead of executing direct SQL count queries.
- Added migration scripts in `sql/mysql/4.3.0.sql` and
`sql/postgresql/4.3.0.sql` to create the totals table and populate it
with existing data.

---
[Ask Sweep AI questions about this PR](https://app.sweep.dev)
<!--- SUMMARY_MARKER --->

> Fix database performance issues caused by doc_count and
doc_version_count queries

1. 💽 Add a corresponding totals table to record statistical information
2. ➕ Add a `PACKAGE_ADDED` event and the original
`PACKAGE_VERSION_ADDED` event to asynchronously update records in the
table
3. ♻️ Add a new existing data migration script to migrate the original
statistical information to the totals table

-----------

> 修复 doc_count 和 doc_version_count 查询导致的数据库性能问题

1. 💽 新增对应 totals 表，用来记录统计信息
2. ➕ 新增 `PACKAGE_ADDED` 事件，和原有 `PACKAGE_VERSION_ADDED` 事件，异步更新表内记录
3. ♻️ 新增存量数据迁移脚本，迁移原有的统计信息到 totals 表

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced persistent tracking of total package and package version
counts, with real-time updates when new packages or versions are added.
- Added new data models and repository methods to manage and retrieve
these total counts.
- Emitted events upon new package creation to update totals
automatically.

- **Database**
- Added a new "totals" table to both MySQL and PostgreSQL databases for
storing aggregate counts initialized from existing data.

- **Bug Fixes**
- Ensured total counts are always returned as numbers in scheduled data
updates.

- **Tests**
- Added and updated tests to verify correct behavior of total count
tracking, incrementing, resetting, and retrieval.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

app/common/enum/Total.ts

@@ -0,0 +1,4 @@
+export enum TotalType {
+  PackageCount = 'packageCount',
+  PackageVersionCount = 'packageVersionCount',
+}

app/core/event/TotalHandler.ts

@@ -0,0 +1,22 @@
+import { Event, Inject } from '@eggjs/tegg';
+import { PACKAGE_ADDED, PACKAGE_VERSION_ADDED } from './index.js';
+import type { TotalRepository } from '../../repository/TotalRepository.js';
+
+class TotalHandlerEvent {
+  @Inject()
+  protected readonly totalRepository: TotalRepository;
+}
+
+@Event(PACKAGE_ADDED)
+export class PackageAddedTotalHandlerEvent extends TotalHandlerEvent {
+  async handle() {
+    await this.totalRepository.incrementPackageCount();
+  }
+}
+
+@Event(PACKAGE_VERSION_ADDED)
+export class PackageVersionAddedTotalHandlerEvent extends TotalHandlerEvent {
+  async handle() {
+    await this.totalRepository.incrementPackageVersionCount();
+  }
+}

app/core/event/index.ts

@@ -1,6 +1,7 @@
 import '@eggjs/tegg';
 import type { User } from '../entity/User.js';
 
+export const PACKAGE_ADDED = 'PACKAGE_ADDED';
 export const PACKAGE_UNPUBLISHED = 'PACKAGE_UNPUBLISHED';
 export const PACKAGE_BLOCKED = 'PACKAGE_BLOCKED';
 export const PACKAGE_UNBLOCKED = 'PACKAGE_UNBLOCKED';
@@ -24,6 +25,7 @@ export interface PackageMetaChange {
 
 declare module '@eggjs/tegg' {
   interface Events {
+    [PACKAGE_ADDED]: (fullname: string) => Promise<void>;
     [PACKAGE_UNPUBLISHED]: (fullname: string) => Promise<void>;
     [PACKAGE_BLOCKED]: (fullname: string) => Promise<void>;
     [PACKAGE_UNBLOCKED]: (fullname: string) => Promise<void>;

app/core/service/PackageManagerService.ts

@@ -41,6 +41,7 @@ import { PackageTag } from '../entity/PackageTag.js';
 import type { User } from '../entity/User.js';
 import type { Dist } from '../entity/Dist.js';
 import {
+  PACKAGE_ADDED,
   PACKAGE_BLOCKED,
   PACKAGE_MAINTAINER_CHANGED,
   PACKAGE_MAINTAINER_REMOVED,
@@ -120,6 +121,7 @@ export class PackageManagerService extends AbstractService {
       await this._checkPackageDepsVersion(cmd.packageJson);
     }
     let pkg = await this.packageRepository.findPackage(cmd.scope, cmd.name);
+    let isNewPackage = !pkg;
     if (pkg) {
       // update description
       // will read database twice to update description by model to entity and entity to model
@@ -337,6 +339,10 @@ export class PackageManagerService extends AbstractService {
       );
     }
 
+    if (isNewPackage) {
+      this.eventBus.emit(PACKAGE_ADDED, pkg.fullname);
+    }
+
     return pkgVersion;
   }
 

app/port/schedule/UpdateTotalData.ts

@@ -150,6 +150,8 @@ export class UpdateTotalData {
     const lastChange = await this.changeRepository.getLastChange();
     const totalData: TotalData = {
       ...packageTotal,
+      packageCount: Number(packageTotal.packageCount),
+      packageVersionCount: Number(packageTotal.packageVersionCount),
       download,
       lastChangeId: (lastChange && lastChange.id) || 0,
       cacheTime: new Date().toISOString(),

app/repository/PackageRepository.ts

@@ -1,12 +1,9 @@
 import { AccessLevel, Inject, SingletonProto } from '@eggjs/tegg';
-import type { Orm } from '@eggjs/tegg-orm-plugin';
-
-import type { Bone } from './util/leoric.js';
-import { Package as PackageModel } from './model/Package.js';
+import type { Package as PackageModel } from './model/Package.js';
 import { Package as PackageEntity } from '../core/entity/Package.js';
 import { ModelConvertor } from './util/ModelConvertor.js';
 import { PackageVersion as PackageVersionEntity } from '../core/entity/PackageVersion.js';
-import { PackageVersion as PackageVersionModel } from './model/PackageVersion.js';
+import type { PackageVersion as PackageVersionModel } from './model/PackageVersion.js';
 import type { PackageVersionManifest as PackageVersionManifestEntity } from '../core/entity/PackageVersionManifest.js';
 import type { PackageVersionManifest as PackageVersionManifestModel } from './model/PackageVersionManifest.js';
 import type { Dist as DistModel } from './model/Dist.js';
@@ -18,6 +15,7 @@ import type { User as UserModel } from './model/User.js';
 import { User as UserEntity } from '../core/entity/User.js';
 import { AbstractRepository } from './AbstractRepository.js';
 import type { BugVersionPackages } from '../core/entity/BugVersion.js';
+import type { TotalRepository } from './TotalRepository.js';
 
 export type PackageManifestType = Pick<PackageJSONType, PackageJSONPickKey> & {
   _id: string;
@@ -227,7 +225,7 @@ export class PackageRepository extends AbstractRepository {
   private readonly User: typeof UserModel;
 
   @Inject()
-  private readonly orm: Orm;
+  private readonly totalRepository: TotalRepository;
 
   async #convertPackageModelToEntity(model: PackageModel) {
     const manifestsDistModel = model.manifestsDistId
@@ -586,18 +584,12 @@ export class PackageRepository extends AbstractRepository {
     );
   }
 
-  private async getTotalCountByModel(model: typeof Bone): Promise<number> {
-    const sql = `SELECT count(id) as total FROM ${model.table};`;
-    const result = await this.orm.client.query(sql);
-    const total = Number(result.rows?.[0].total);
-    return total;
-  }
-
   public async queryTotal() {
+    const { packageCount, packageVersionCount } =
+      await this.totalRepository.getAll();
+
     const lastPkg = await this.Package.findOne().order('id', 'desc');
     const lastVersion = await this.PackageVersion.findOne().order('id', 'desc');
-    let packageCount = 0;
-    let packageVersionCount = 0;
     let lastPackage = '';
     let lastPackageVersion = '';
 
@@ -605,9 +597,6 @@ export class PackageRepository extends AbstractRepository {
       lastPackage = lastPkg.scope
         ? `${lastPkg.scope}/${lastPkg.name}`
         : lastPkg.name;
-      // FIXME: id will be out of range number
-      // 可能存在 id 增长不连续的情况，通过 count 查询
-      packageCount = await this.getTotalCountByModel(PackageModel);
     }
 
     if (lastVersion) {
@@ -618,12 +607,11 @@ export class PackageRepository extends AbstractRepository {
         const fullname = pkg.scope ? `${pkg.scope}/${pkg.name}` : pkg.name;
         lastPackageVersion = `${fullname}@${lastVersion.version}`;
       }
-      packageVersionCount =
-        await this.getTotalCountByModel(PackageVersionModel);
     }
+
     return {
-      packageCount,
-      packageVersionCount,
+      packageCount: Number(packageCount),
+      packageVersionCount: Number(packageVersionCount),
       lastPackage,
       lastPackageVersion,
     };

app/repository/TotalRepository.ts

@@ -0,0 +1,73 @@
+import { AccessLevel, Inject, SingletonProto } from '@eggjs/tegg';
+import type { Total } from './model/Total.js';
+import { AbstractRepository } from './AbstractRepository.js';
+import { TotalType } from '../common/enum/Total.js';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class TotalRepository extends AbstractRepository {
+  @Inject()
+  private readonly Total: typeof Total;
+
+  // Package count methods
+  async incrementPackageCount(count = 1) {
+    await this.increment(TotalType.PackageCount, count);
+  }
+
+  async getPackageCount(): Promise<number> {
+    return this.get(TotalType.PackageCount);
+  }
+
+  // Package version count methods
+  async incrementPackageVersionCount(count = 1) {
+    await this.increment(TotalType.PackageVersionCount, count);
+  }
+
+  async getPackageVersionCount(): Promise<number> {
+    return this.get(TotalType.PackageVersionCount);
+  }
+
+  // Private helper methods
+  private async increment(type: TotalType, count = 1) {
+    const model = await this.Total.findOne({ type });
+    if (model) {
+      await this.Total.where({ id: model.id }).increment('count', count);
+    } else {
+      await this.Total.create({
+        type,
+        count: BigInt(count),
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      });
+    }
+  }
+
+  private async get(type: TotalType): Promise<number> {
+    const model = await this.Total.findOne({ type });
+    return model ? Number(model.count.toString()) : 0;
+  }
+
+  // Get all counts
+  async getAll(): Promise<{
+    packageCount: string;
+    packageVersionCount: string;
+  }> {
+    const [packageCount, packageVersionCount] = await Promise.all([
+      this.getPackageCount(),
+      this.getPackageVersionCount(),
+    ]);
+    return {
+      packageCount: packageCount.toString(),
+      packageVersionCount: packageVersionCount.toString(),
+    };
+  }
+
+  // Reset all counters to 0
+  async reset() {
+    await this.Total.where({}).update({
+      count: '0',
+      updatedAt: new Date(),
+    });
+  }
+}

app/repository/model/Total.ts

@@ -0,0 +1,26 @@
+import { Attribute, Model } from '@eggjs/tegg/orm';
+
+import { Bone, DataTypes } from '../util/leoric.js';
+
+@Model()
+export class Total extends Bone {
+  @Attribute(DataTypes.BIGINT, {
+    primary: true,
+    autoIncrement: true,
+  })
+  id: bigint;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_create' })
+  createdAt: Date;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_modified' })
+  updatedAt: Date;
+
+  @Attribute(DataTypes.STRING(24), {
+    unique: true,
+  })
+  type: string;
+
+  @Attribute(DataTypes.BIGINT)
+  count: bigint;
+}

sql/ddl_mysql.sql

@@ -432,3 +432,13 @@ CREATE TABLE `webauthn_credentials` (
   KEY `idx_user_id` (`user_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8_unicode_ci COMMENT='webauthn credential info'
 ;
+
+CREATE TABLE IF NOT EXISTS `totals` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `type` varchar(24) NOT NULL COMMENT 'total type',
+  `count` bigint(20) NOT NULL COMMENT 'total count',
+  `gmt_create` datetime NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_type` (`type`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='total table';

sql/ddl_postgresql.sql

@@ -702,3 +702,19 @@ COMMENT ON COLUMN webauthn_credentials.user_id IS 'user id';
 COMMENT ON COLUMN webauthn_credentials.credential_id IS 'webauthn credential id';
 COMMENT ON COLUMN webauthn_credentials.public_key IS 'webauthn credential publick key';
 COMMENT ON COLUMN webauthn_credentials.browser_type IS 'user browser name';
+
+CREATE TABLE IF NOT EXISTS totals (
+  id BIGSERIAL PRIMARY KEY,
+  gmt_create timestamp(3) NOT NULL,
+  gmt_modified timestamp(3) NOT NULL,
+  type varchar(24) NOT NULL,
+  count bigint NOT NULL,
+  CONSTRAINT uk_type UNIQUE (type)
+);
+
+COMMENT ON TABLE totals IS 'total table';
+COMMENT ON COLUMN totals.id IS 'primary key';
+COMMENT ON COLUMN totals.type IS 'total type';
+COMMENT ON COLUMN totals.count IS 'total count';
+COMMENT ON COLUMN totals.gmt_create IS 'create time';
+COMMENT ON COLUMN totals.gmt_modified IS 'modified time';

sql/mysql/4.3.0.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS `totals` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `type` varchar(24) NOT NULL COMMENT 'total type',
+  `count` bigint(20) NOT NULL COMMENT 'total count',
+  `gmt_create` datetime NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_type` (`type`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='total table';
+
+-- init data
+INSERT INTO `totals` (`type`, `count`, `gmt_create`, `gmt_modified`)
+SELECT 'packageCount', COUNT(*), NOW(), NOW() FROM `packages`
+UNION ALL
+SELECT 'packageVersionCount', COUNT(*), NOW(), NOW() FROM `package_versions`;

sql/postgresql/4.3.0.sql

@@ -0,0 +1,20 @@
+CREATE TABLE IF NOT EXISTS totals (
+  id BIGSERIAL PRIMARY KEY,
+  gmt_create timestamp(3) NOT NULL,
+  gmt_modified timestamp(3) NOT NULL,
+  type varchar(24) NOT NULL,
+  count bigint NOT NULL,
+  CONSTRAINT uk_type UNIQUE (type)
+);
+
+COMMENT ON TABLE totals IS 'total table';
+COMMENT ON COLUMN totals.id IS 'primary key';
+COMMENT ON COLUMN totals.type IS 'total type';
+COMMENT ON COLUMN totals.count IS 'total count';
+COMMENT ON COLUMN totals.gmt_create IS 'create time';
+COMMENT ON COLUMN totals.gmt_modified IS 'modified time';
+
+INSERT INTO totals (type, count, gmt_create, gmt_modified)
+SELECT 'packageCount', COUNT(*), NOW(), NOW() FROM packages
+UNION ALL
+SELECT 'packageVersionCount', COUNT(*), NOW(), NOW() FROM package_versions;

test/port/controller/HomeController/showTotal.test.ts

@@ -15,6 +15,7 @@ import type { ChangesStreamTask } from '../../../../app/core/entity/Task.js';
 import { RegistryType } from '../../../../app/common/enum/Registry.js';
 import { ScopeManagerService } from '../../../../app/core/service/ScopeManagerService.js';
 import type { UpstreamRegistryInfo } from '../../../../app/core/service/CacheService.js';
+import { TotalRepository } from '../../../../app/repository/TotalRepository.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -33,8 +34,11 @@ describe('test/port/controller/HomeController/showTotal.test.ts', () => {
     let registryManagerService: RegistryManagerService;
     let changesStreamService: ChangesStreamService;
     let taskRepository: TaskRepository;
+    let totalRepository: TotalRepository;
     let scopeManagerService: ScopeManagerService;
     it('should total information', async () => {
+      totalRepository = await app.getEggObject(TotalRepository);
+      await totalRepository.reset();
       let res = await app.httpRequest().get('/');
       assert(res.status === 200);
       assert(res.headers['content-type'] === 'application/json; charset=utf-8');
@@ -63,10 +67,12 @@ describe('test/port/controller/HomeController/showTotal.test.ts', () => {
         .set('user-agent', publisher.ua)
         .expect(201)
         .send(pkg);
+
       pkg = await TestUtil.getFullPackage({
         name: '@cnpm/home2',
         version: '2.0.0',
       });
+
       await app
         .httpRequest()
         .put(`/${pkg.name}`)
@@ -74,6 +80,7 @@ describe('test/port/controller/HomeController/showTotal.test.ts', () => {
         .set('user-agent', publisher.ua)
         .expect(201)
         .send(pkg);
+
       pkg = await TestUtil.getFullPackage({
         name: '@cnpm/home1',
         version: '1.0.1',

test/repository/PackageRepository.test.ts

@@ -5,6 +5,7 @@ import { PackageRepository } from '../../app/repository/PackageRepository.js';
 import { PackageManagerService } from '../../app/core/service/PackageManagerService.js';
 import { UserService } from '../../app/core/service/UserService.js';
 import { TestUtil } from '../../test/TestUtil.js';
+import { setTimeout } from 'node:timers/promises';
 
 describe('test/repository/PackageRepository.test.ts', () => {
   let packageRepository: PackageRepository;
@@ -42,9 +43,9 @@ describe('test/repository/PackageRepository.test.ts', () => {
         },
         user
       );
+      await setTimeout(1000);
       const res = await packageRepository.queryTotal();
-      // information_schema 只能返回大概值，仅验证增加
-      assert(res.packageCount > packageCount);
+      assert(res.packageCount >= packageCount);
       assert(res.packageVersionCount > packageVersionCount);
     });
   });

test/repository/TotalRepository.test.ts

@@ -0,0 +1,50 @@
+import assert from 'node:assert/strict';
+import { app } from '@eggjs/mock/bootstrap';
+
+import { TotalRepository } from '../../app/repository/TotalRepository.js';
+
+describe('test/repository/TotalRepository.test.ts', () => {
+  let totalRepository: TotalRepository;
+
+  beforeEach(async () => {
+    totalRepository = await app.getEggObject(TotalRepository);
+  });
+
+  describe('TotalRepository', () => {
+    it('should get initial package count', async () => {
+      const count = await totalRepository.getPackageCount();
+      assert.equal(count, 0);
+    });
+
+    it('should get initial package version count', async () => {
+      const count = await totalRepository.getPackageVersionCount();
+      assert.equal(count, 0);
+    });
+
+    it('should increment package count', async () => {
+      await totalRepository.incrementPackageCount();
+      const count = await totalRepository.getPackageCount();
+      assert.equal(count, 1);
+    });
+
+    it('should increment package version count', async () => {
+      await totalRepository.incrementPackageVersionCount();
+      const count = await totalRepository.getPackageVersionCount();
+      assert.equal(count, 1);
+    });
+
+    it('should increment multiple times', async () => {
+      await totalRepository.incrementPackageCount();
+      await totalRepository.incrementPackageCount();
+      await totalRepository.incrementPackageVersionCount();
+      await totalRepository.incrementPackageVersionCount();
+      await totalRepository.incrementPackageVersionCount();
+
+      const packageCount = await totalRepository.getPackageCount();
+      const versionCount = await totalRepository.getPackageVersionCount();
+
+      assert.equal(packageCount, 2);
+      assert.equal(versionCount, 3);
+    });
+  });
+});