liaohui
/
fenghuo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add

This commit is contained in:
ditiqi 2025-05-29 10:27:11 +08:00
parent 8938337944
commit 047e1fb80a
13 changed files with 367 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/backend/src/adapters/index.ts Normal file
View File

@ -0,0 +1 @@
export * from './storage-adapter';

106
apps/backend/src/adapters/storage-adapter.ts Normal file
View File

@ -0,0 +1,106 @@
import { prisma } from '@repo/db';
import type { Resource } from '@repo/db';
import type { DatabaseAdapter, StorageType, ResourceData, CreateResourceData } from '@repo/storage';
// 将 Prisma Resource 转换为 ResourceData 的辅助函数
function transformResource(resource: Resource): ResourceData {
return {
id: resource.id,
fileId: resource.fileId,
title: resource.title,
type: resource.type,
storageType: resource.storageType as StorageType,
status: resource.status || 'unknown',
meta: resource.meta,
createdAt: resource.createdAt,
updatedAt: resource.updatedAt,
};
}
export class PrismaDatabaseAdapter implements DatabaseAdapter {
async getResourceByFileId(fileId: string): Promise<{ status: string; resource?: ResourceData }> {
const resource = await prisma.resource.findFirst({
where: { fileId },
});
if (!resource) {
return { status: 'pending' };
}
return {
status: resource.status || 'unknown',
resource: transformResource(resource),
};
}
async deleteResource(id: string): Promise<ResourceData> {
const resource = await prisma.resource.delete({
where: { id },
});
return transformResource(resource);
}
async deleteFailedUploadingResource(expirationPeriod: number): Promise<{ count: number }> {
const deletedResources = await prisma.resource.deleteMany({
where: {
createdAt: {
lt: new Date(Date.now() - expirationPeriod),
},
status: 'UPLOADING',
},
});
return deletedResources;
}
async updateResource(id: string, data: any): Promise<ResourceData> {
const resource = await prisma.resource.update({
where: { id },
data,
});
return transformResource(resource);
}
async migrateResourcesStorageType(
fromStorageType: StorageType,
toStorageType: StorageType,
): Promise<{ count: number }> {
const result = await prisma.resource.updateMany({
where: {
storageType: fromStorageType,
},
data: {
storageType: toStorageType,
},
});
return { count: result.count };
}
async createResource(data: CreateResourceData): Promise<ResourceData> {
const resource = await prisma.resource.create({
data: {
fileId: data.fileId,
title: data.filename,
type: data.mimeType,
storageType: data.storageType,
status: data.status || 'UPLOADING',
meta: {
size: data.size,
hash: data.hash,
},
},
});
return transformResource(resource);
}
async updateResourceStatus(fileId: string, status: string, additionalData?: any): Promise<ResourceData> {
const resource = await prisma.resource.update({
where: { fileId },
data: {
status,
...additionalData,
},
});
return transformResource(resource);
}
}

7
apps/backend/src/index.ts
View File

@ -18,7 +18,12 @@ import { wsHandler, wsConfig } from './socket';
// 导入新的路由
import userRest from './user/user.rest';
// 使用新的 @repo/storage 包
import { createStorageApp, startCleanupScheduler } from '@repo/storage';
import { createStorageApp, startCleanupScheduler, adapterRegistry } from '@repo/storage';
// 导入 Prisma 适配器实现
import { PrismaDatabaseAdapter } from './adapters/storage-adapter';
// 注册数据库适配器
adapterRegistry.setDatabaseAdapter(new PrismaDatabaseAdapter());
type Env = {
Variables: {

193
packages/storage/README.md
View File

@ -11,6 +11,11 @@
- 🗄️ **数据库集成**: 与 Prisma 深度集成
- ⏰ **自动清理**: 支持过期文件自动清理
- 🔄 **存储迁移**: 支持不同存储类型间的数据迁移
- 🔌 **适配器模式** - 通过适配器与任何数据库后端集成
- 📁 **多存储后端** - 支持本地存储和 S3 兼容存储
- 🚀 **TUS 协议** - 支持可恢复文件上传
- 🔄 **自动清理** - 自动清理失败的上传
- 🛡️ **类型安全** - 完整的 TypeScript 支持
## 安装
@ -112,91 +117,163 @@ S3_FORCE_PATH_STYLE=false
## 快速开始
### 1. 基础使用
### 1. 安装依赖
```bash
npm install @repo/storage
```
### 2. 实现数据库适配器
```typescript
import { DatabaseAdapter, ResourceData, CreateResourceData, StorageType } from '@repo/storage';
export class MyDatabaseAdapter implements DatabaseAdapter {
async getResourceByFileId(fileId: string): Promise<{ status: string; resource?: ResourceData }> {
// 实现从数据库获取资源的逻辑
}
async createResource(data: CreateResourceData): Promise<ResourceData> {
// 实现创建资源的逻辑
}
async updateResource(id: string, data: any): Promise<ResourceData> {
// 实现更新资源的逻辑
}
async deleteResource(id: string): Promise<ResourceData> {
// 实现删除资源的逻辑
}
async updateResourceStatus(fileId: string, status: string, additionalData?: any): Promise<ResourceData> {
// 实现更新资源状态的逻辑
}
async deleteFailedUploadingResource(expirationPeriod: number): Promise<{ count: number }> {
// 实现清理失败上传的逻辑
}
async migrateResourcesStorageType(
fromStorageType: StorageType,
toStorageType: StorageType,
): Promise<{ count: number }> {
// 实现存储类型迁移的逻辑
}
}
```
### 3. 注册适配器
```typescript
import { adapterRegistry } from '@repo/storage';
import { MyDatabaseAdapter } from './my-database-adapter';
// 在应用启动时注册适配器
adapterRegistry.setDatabaseAdapter(new MyDatabaseAdapter());
```
### 4. 使用存储功能
```typescript
import { createStorageApp, startCleanupScheduler } from '@repo/storage';
import { Hono } from 'hono';
const app = new Hono();
// 创建存储应用
const storageApp = createStorageApp({
apiBasePath: '/api/storage', // API 路径
uploadPath: '/upload', // 上传路径
apiBasePath: '/api/storage',
uploadPath: '/upload',
});
// 挂载存储应用
app.route('/', storageApp);
// 启动清理调度器
// 启动清理任务
startCleanupScheduler();
```
### 2. 分别使用 API 和上传功能
## Prisma 适配器示例
如果您使用 Prisma，可以参考以下实现：
```typescript
import { createStorageRoutes, createTusUploadRoutes } from '@repo/storage';
import { prisma } from '@your/db-package';
import { DatabaseAdapter, ResourceData, CreateResourceData, StorageType } from '@repo/storage';
const app = new Hono();
export class PrismaDatabaseAdapter implements DatabaseAdapter {
// 将 Prisma Resource 转换为 ResourceData
private transformResource(resource: any): ResourceData {
return {
id: resource.id,
fileId: resource.fileId,
title: resource.title,
type: resource.type,
storageType: resource.storageType as StorageType,
status: resource.status || 'unknown',
meta: resource.meta,
createdAt: resource.createdAt,
updatedAt: resource.updatedAt,
};
}
// 只添加存储管理 API
app.route('/api/storage', createStorageRoutes());
async getResourceByFileId(fileId: string): Promise<{ status: string; resource?: ResourceData }> {
const resource = await prisma.resource.findFirst({
where: { fileId },
});
// 只添加文件上传功能
app.route('/upload', createTusUploadRoutes());
```
if (!resource) {
return { status: 'pending' };
}
### 3. 使用存储管理器
return {
status: resource.status || 'unknown',
resource: this.transformResource(resource),
};
}
```typescript
import { StorageManager, StorageUtils } from '@repo/storage';
async createResource(data: CreateResourceData): Promise<ResourceData> {
const resource = await prisma.resource.create({
data: {
fileId: data.fileId,
title: data.filename,
type: data.mimeType,
storageType: data.storageType,
status: data.status || 'UPLOADING',
meta: {
size: data.size,
hash: data.hash,
},
},
});
return this.transformResource(resource);
}
// 获取存储管理器实例
const storageManager = StorageManager.getInstance();
// 获取存储信息
const storageInfo = storageManager.getStorageInfo();
console.log('当前存储类型:', storageInfo.type);
// 使用存储工具
const storageUtils = StorageUtils.getInstance();
// 生成文件访问 URL（统一使用下载接口）
const fileUrl = storageUtils.generateFileUrl('2024/01/01/abc123/file.jpg');
// 结果: http://localhost:3000/download/2024/01/01/abc123/file.jpg
// 生成完整的公开访问 URL
const publicUrl = storageUtils.generateFileUrl('2024/01/01/abc123/file.jpg', 'https://yourdomain.com');
// 结果: https://yourdomain.com/download/2024/01/01/abc123/file.jpg
// 生成 S3 直接访问 URL（仅 S3 存储）
try {
const directUrl = storageUtils.generateDirectUrl('2024/01/01/abc123/file.jpg');
// S3 存储: https://bucket.s3.region.amazonaws.com/2024/01/01/abc123/file.jpg
} catch (error) {
// 本地存储会抛出错误
// ... 实现其他方法
}
// 检查文件是否存在
const exists = await storageUtils.fileExists('file-id');
```
### 4. 分别配置不同功能
## API 参考
### DatabaseAdapter 接口
所有数据库适配器都必须实现 `DatabaseAdapter` 接口：
- `getResourceByFileId(fileId: string)` - 根据文件ID获取资源
- `createResource(data: CreateResourceData)` - 创建新资源
- `updateResource(id: string, data: any)` - 更新资源
- `deleteResource(id: string)` - 删除资源
- `updateResourceStatus(fileId: string, status: string, additionalData?: any)` - 更新资源状态
- `deleteFailedUploadingResource(expirationPeriod: number)` - 清理失败的上传
- `migrateResourcesStorageType(from: StorageType, to: StorageType)` - 迁移存储类型
### 适配器注册器
```typescript
import { createStorageRoutes, createTusUploadRoutes, createFileDownloadRoutes } from '@repo/storage';
import { adapterRegistry } from '@repo/storage';
const app = new Hono();
// 注册适配器
adapterRegistry.setDatabaseAdapter(adapter);
// 只添加存储管理 API
app.route('/api/storage', createStorageRoutes());
// 获取当前适配器
const adapter = adapterRegistry.getDatabaseAdapter();
// 只添加文件上传功能
app.route('/upload', createTusUploadRoutes());
// 只添加文件下载功能（所有存储类型）
app.route('/download', createFileDownloadRoutes());
// 检查是否已注册适配器
const hasAdapter = adapterRegistry.hasAdapter();
```
## API 端点

26
packages/storage/src/adapters/adapter-registry.ts Normal file
View File

@ -0,0 +1,26 @@
import type { DatabaseAdapter } from './database-adapter';
class AdapterRegistry {
private _adapter: DatabaseAdapter | null = null;
// 注册数据库适配器
setDatabaseAdapter(adapter: DatabaseAdapter): void {
this._adapter = adapter;
}
// 获取数据库适配器
getDatabaseAdapter(): DatabaseAdapter {
if (!this._adapter) {
throw new Error('数据库适配器未注册。请在使用存储功能前调用 setDatabaseAdapter() 注册适配器。');
}
return this._adapter;
}
// 检查是否已注册适配器
hasAdapter(): boolean {
return this._adapter !== null;
}
}
// 导出单例实例
export const adapterRegistry = new AdapterRegistry();

15
packages/storage/src/adapters/database-adapter.ts Normal file
View File

@ -0,0 +1,15 @@
import type { StorageType, ResourceData, CreateResourceData } from '../types';
// 数据库适配器接口 - 基于 operations.ts 中的函数
export interface DatabaseAdapter {
getResourceByFileId(fileId: string): Promise<{ status: string; resource?: ResourceData }>;
deleteResource(id: string): Promise<ResourceData>;
deleteFailedUploadingResource(expirationPeriod: number): Promise<{ count: number }>;
updateResource(id: string, data: any): Promise<ResourceData>;
migrateResourcesStorageType(
fromStorageType: StorageType,
toStorageType: StorageType,
): Promise<{ count: number }>;
createResource(data: CreateResourceData): Promise<ResourceData>;
updateResourceStatus(fileId: string, status: string, additionalData?: any): Promise<ResourceData>;
}

5
packages/storage/src/adapters/index.ts Normal file
View File

@ -0,0 +1,5 @@
// 数据库适配器接口
export * from './database-adapter';
// 适配器注册器
export * from './adapter-registry';

161
packages/storage/src/database/operations.ts
View File

@ -1,153 +1,44 @@
import { prisma } from '@repo/db';
import type { Resource } from '@repo/db';
import { StorageType } from '../types';
import { adapterRegistry } from '../adapters/adapter-registry';
import type { StorageType, ResourceData, CreateResourceData } from '../types';
export async function getResourceByFileId(fileId: string): Promise<{ status: string; resource?: Resource }> {
const resource = await prisma.resource.findFirst({
where: { fileId },
});
if (!resource) {
return { status: 'pending' };
}
return {
status: resource.status || 'unknown',
resource,
};
export async function getResourceByFileId(fileId: string): Promise<{ status: string; resource?: ResourceData }> {
const adapter = adapterRegistry.getDatabaseAdapter();
return adapter.getResourceByFileId(fileId);
}
export async function getAllResources(): Promise<Resource[]> {
return prisma.resource.findMany({
orderBy: { createdAt: 'desc' },
});
export async function deleteResource(id: string): Promise<ResourceData> {
const adapter = adapterRegistry.getDatabaseAdapter();
return adapter.deleteResource(id);
}
export async function getResourcesByStorageType(storageType: StorageType): Promise<Resource[]> {
return prisma.resource.findMany({
where: {
storageType: storageType,
},
orderBy: { createdAt: 'desc' },
});
export async function deleteFailedUploadingResource(expirationPeriod: number): Promise<{ count: number }> {
const adapter = adapterRegistry.getDatabaseAdapter();
return adapter.deleteFailedUploadingResource(expirationPeriod);
}
export async function getResourcesByStatus(status: string): Promise<Resource[]> {
return prisma.resource.findMany({
where: { status },
orderBy: { createdAt: 'desc' },
});
}
export async function getUploadingResources(): Promise<Resource[]> {
return prisma.resource.findMany({
where: {
status: 'UPLOADING',
},
orderBy: { createdAt: 'desc' },
});
}
export async function getResourceStats(): Promise<{
total: number;
byStatus: Record<string, number>;
byStorageType: Record<string, number>;
}> {
const [total, statusStats, storageStats] = await Promise.all([
prisma.resource.count(),
prisma.resource.groupBy({
by: ['status'],
_count: true,
}),
prisma.resource.groupBy({
by: ['storageType'],
_count: true,
}),
]);
const byStatus = statusStats.reduce(
(acc, item) => {
acc[item.status || 'unknown'] = item._count || 0;
return acc;
},
{} as Record<string, number>,
);
const byStorageType = storageStats.reduce(
(acc, item) => {
const key = (item.storageType as string) || 'unknown';
acc[key] = item._count;
return acc;
},
{} as Record<string, number>,
);
return {
total,
byStatus,
byStorageType,
};
}
export async function deleteResource(id: string): Promise<Resource> {
return prisma.resource.delete({
where: { id },
});
}
export async function updateResource(id: string, data: any): Promise<Resource> {
return prisma.resource.update({
where: { id },
data,
});
export async function updateResource(id: string, data: any): Promise<ResourceData> {
const adapter = adapterRegistry.getDatabaseAdapter();
return adapter.updateResource(id, data);
}
export async function migrateResourcesStorageType(
fromStorageType: StorageType,
toStorageType: StorageType,
): Promise<{ count: number }> {
const result = await prisma.resource.updateMany({
where: {
storageType: fromStorageType,
},
data: {
storageType: toStorageType,
},
});
return { count: result.count };
const adapter = adapterRegistry.getDatabaseAdapter();
return adapter.migrateResourcesStorageType(fromStorageType, toStorageType);
}
export async function createResource(data: {
fileId: string;
filename: string;
size: number;
mimeType?: string | null;
storageType: StorageType;
status?: string;
hash?: string;
}): Promise<Resource> {
return prisma.resource.create({
data: {
fileId: data.fileId,
title: data.filename,
type: data.mimeType,
storageType: data.storageType,
status: data.status || 'UPLOADING',
meta: {
size: data.size,
hash: data.hash,
},
},
});
export async function createResource(data: CreateResourceData): Promise<ResourceData> {
const adapter = adapterRegistry.getDatabaseAdapter();
return adapter.createResource(data);
}
export async function updateResourceStatus(fileId: string, status: string, additionalData?: any): Promise<Resource> {
return prisma.resource.update({
where: { fileId },
data: {
status,
...additionalData,
},
});
export async function updateResourceStatus(
fileId: string,
status: string,
additionalData?: any,
): Promise<ResourceData> {
const adapter = adapterRegistry.getDatabaseAdapter();
return adapter.updateResourceStatus(fileId, status, additionalData);
}

15
packages/storage/src/enum.ts Normal file
View File

@ -0,0 +1,15 @@
export enum QueueJobType {
UPDATE_STATS = 'update_stats',
FILE_PROCESS = 'file_process',
UPDATE_POST_VISIT_COUNT = 'updatePostVisitCount',
UPDATE_POST_STATE = 'updatePostState',
}
export enum ResourceStatus {
UPLOADING = 'UPLOADING',
UPLOADED = 'UPLOADED',
PROCESS_PENDING = 'PROCESS_PENDING',
PROCESSING = 'PROCESSING',
PROCESSED = 'PROCESSED',
PROCESS_FAILED = 'PROCESS_FAILED',
}

5
packages/storage/src/index.ts
View File

@ -12,6 +12,10 @@ export * from './services';
// Hono 中间件
export * from './middleware';
export * from './enum';
// 适配器系统
export * from './adapters';
// TUS 协议支持 (已集成)
// TUS 相关功能通过 services 层提供，如需直接访问 TUS 类，可使用：
@ -24,3 +28,4 @@ export { StorageUtils } from './services';
export { getTusServer, handleTusRequest } from './services';
export { startCleanupScheduler, triggerCleanup } from './services';
export { createStorageApp, createStorageRoutes, createTusUploadRoutes, createFileDownloadRoutes } from './middleware';
export { adapterRegistry } from './adapters/adapter-registry';

70
packages/storage/src/middleware/hono.ts
View File

@ -2,18 +2,12 @@ import { Hono } from 'hono';
import { handleTusRequest, cleanupExpiredUploads, getStorageInfo } from '../services/tus';
import {
getResourceByFileId,
getAllResources,
deleteResource,
updateResource,
getResourcesByStorageType,
getResourcesByStatus,
getUploadingResources,
getResourceStats,
migrateResourcesStorageType,
} from '../database/operations';
import { StorageManager, validateStorageConfig } from '../core/adapter';
import { StorageType, type StorageConfig } from '../types';
import { prisma } from '@repo/db';
/**
* 创建存储相关的 Hono 路由
@ -33,37 +27,6 @@ export function createStorageRoutes(basePath: string = '/api/storage') {
return c.json(result);
});
// 获取所有资源
app.get('/resources', async (c) => {
const resources = await getAllResources();
return c.json(resources);
});
// 根据存储类型获取资源
app.get('/resources/storage/:storageType', async (c) => {
const storageType = c.req.param('storageType') as StorageType;
const resources = await getResourcesByStorageType(storageType);
return c.json(resources);
});
// 根据状态获取资源
app.get('/resources/status/:status', async (c) => {
const status = c.req.param('status');
const resources = await getResourcesByStatus(status);
return c.json(resources);
});
// 获取正在上传的资源
app.get('/resources/uploading', async (c) => {
const resources = await getUploadingResources();
return c.json(resources);
});
// 获取资源统计信息
app.get('/stats', async (c) => {
const stats = await getResourceStats();
return c.json(stats);
});
// 删除资源
app.delete('/resource/:id', async (c) => {
@ -108,39 +71,6 @@ export function createStorageRoutes(basePath: string = '/api/storage') {
return c.json(result);
});
// 手动清理指定状态的资源
app.post('/cleanup/by-status', async (c) => {
try {
const { status, olderThanDays } = await c.req.json();
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - (olderThanDays || 30));
const deletedResources = await prisma.resource.deleteMany({
where: {
status,
createdAt: {
lt: cutoffDate,
},
},
});
return c.json({
success: true,
message: `Deleted ${deletedResources.count} resources with status ${status}`,
count: deletedResources.count,
});
} catch (error) {
console.error('Failed to cleanup by status:', error);
return c.json(
{
success: false,
error: error instanceof Error ? error.message : 'Unknown error',
},
400,
);
}
});
// 获取存储信息
app.get('/storage/info', async (c) => {
const storageInfo = getStorageInfo();

29
packages/storage/src/services/tus.ts
View File

@ -1,30 +1,14 @@
import { Server, Upload } from '../tus';
import { prisma } from '@repo/db';
import { nanoid } from 'nanoid';
import { slugify } from 'transliteration';
import { StorageManager } from '../core/adapter';
import { createResource, updateResourceStatus } from '../database/operations';
import { createResource, deleteFailedUploadingResource, updateResourceStatus } from '../database/operations';
import { ResourceStatus } from '../enum';
const FILE_UPLOAD_CONFIG = {
maxSizeBytes: 20_000_000_000, // 20GB
};
export enum QueueJobType {
UPDATE_STATS = 'update_stats',
FILE_PROCESS = 'file_process',
UPDATE_POST_VISIT_COUNT = 'updatePostVisitCount',
UPDATE_POST_STATE = 'updatePostState',
}
export enum ResourceStatus {
UPLOADING = 'UPLOADING',
UPLOADED = 'UPLOADED',
PROCESS_PENDING = 'PROCESS_PENDING',
PROCESSING = 'PROCESSING',
PROCESSED = 'PROCESSED',
PROCESS_FAILED = 'PROCESS_FAILED',
}
// 全局 TUS 服务器实例
let tusServer: Server | null = null;
@ -149,14 +133,7 @@ export async function cleanupExpiredUploads() {
const expirationPeriod: number = 24 * 60 * 60 * 1000;
// Delete incomplete uploads older than expiration period
const deletedResources = await prisma.resource.deleteMany({
where: {
createdAt: {
lt: new Date(Date.now() - expirationPeriod),
},
status: ResourceStatus.UPLOADING,
},
});
const deletedResources = await deleteFailedUploadingResource(expirationPeriod);
const server = getTusServer();
const expiredUploadCount = await server.cleanUpExpiredUploads();

24
packages/storage/src/types/index.ts
View File

@ -49,3 +49,27 @@ export interface StorageConfig {
expirationPeriodInMilliseconds?: number;
};
}
// 资源数据接口
export interface ResourceData {
id: string;
fileId: string;
title: string;
type?: string | null;
storageType: StorageType;
status: string;
meta?: any;
createdAt: Date;
updatedAt: Date;
}
// 创建资源数据接口
export interface CreateResourceData {
fileId: string;
filename: string;
size: number;
mimeType?: string | null;
storageType: StorageType;
status?: string;
hash?: string;
}