diff --git a/.env.demo b/.env.demo index f5de4cd3c..0c24dd0b5 100644 --- a/.env.demo +++ b/.env.demo @@ -109,24 +109,37 @@ DATABASE_URL="postgresql://postgres:postgres@your-ip:5432/credebl" AWS_ACCESS_KEY= AWS_SECRET_KEY= AWS_REGION= -AWS_BUCKET= +FILE_SHARING_BUCKET= # Used for Adding org-logo during org creation and update (Optional- Can be skipped if no image is added during org creation and updation) # Provide aws bucket access credentails with write access to the bucket as logo upload is done using signed url which requires putObject permission. AWS_PUBLIC_ACCESS_KEY= AWS_PUBLIC_SECRET_KEY= AWS_PUBLIC_REGION= -AWS_ORG_LOGO_BUCKET_NAME= +ORG_LOGO_BUCKET_NAME= # Used for storing connection URL generated from Agent and creating shortened URL (As connecting to org requires Shortened url) # Provide aws bucket access credentails with write access to the bucket as storing shortened url is done using putObject operation. AWS_S3_STOREOBJECT_ACCESS_KEY= AWS_S3_STOREOBJECT_SECRET_KEY= AWS_S3_STOREOBJECT_REGION= -AWS_S3_STOREOBJECT_BUCKET= +STOREOBJECT_BUCKET= # Provide bucket used for storing shortened url objects e.g. 'https://s3.region-code.amazonaws.com/bucket-name' -SHORTENED_URL_DOMAIN='https://s3.AWS_S3_STOREOBJECT_REGION.amazonaws.com/AWS_S3_STOREOBJECT_BUCKET' +SHORTENED_URL_DOMAIN='https://s3.AWS_S3_STOREOBJECT_REGION.amazonaws.com/STOREOBJECT_BUCKET' + +# Storage type: 'aws' (default), 'rustfs' (S3-compatible), or 'local' (filesystem) +# When set to 'local', AWS/RustFS credentials are not required +FILE_STORAGE_TYPE=aws + +RUSTFS_ENDPOINT= +RUSTFS_ACCESS_KEY_ID= +RUSTFS_SECRET_ACCESS_KEY= +RUSTFS_REGION= +# make sure to add the bucket names in the below variables if you are using local rustfs for file storage, as these buckets need to be created in rustfs and the same bucket names should be used in the code for storing and retrieving files. +# FILE_SHARING_BUCKET= +# ORG_LOGO_BUCKET_NAME= +# STOREOBJECT_BUCKET= # Specify your domain/subdomain responsible for deeplinking with 'url' as a query param e.g. 'https://your-deeplink-domain?url=' DEEPLINK_DOMAIN='https://link.credebl.id?url=' diff --git a/.gitignore b/.gitignore index 4b82df9a5..82f366ee3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ sonar-project.properties .scannerwork/* coverage libs/prisma-service/prisma/data/credebl-master-table/credebl-master-table.json +uploadedFiles/* uploadedFles/exports uploadedFles/import uploadedFles/export diff --git a/apps/api-gateway/src/issuance/issuance.controller.ts b/apps/api-gateway/src/issuance/issuance.controller.ts index be351d956..5b39804d9 100644 --- a/apps/api-gateway/src/issuance/issuance.controller.ts +++ b/apps/api-gateway/src/issuance/issuance.controller.ts @@ -69,7 +69,7 @@ import { IssueCredentialType, UploadedFileDetails } from './interfaces'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { FileInterceptor } from '@nestjs/platform-express'; import { v4 as uuidv4 } from 'uuid'; import { RpcException } from '@nestjs/microservices'; @@ -90,7 +90,7 @@ import { IWebhookUrlInfo } from '@credebl/common/interfaces/webhook.interface'; export class IssuanceController { constructor( private readonly issueCredentialService: IssuanceService, - private readonly awsService: AwsService + private readonly awsService: StorageService ) {} private readonly logger = new Logger('IssuanceController'); diff --git a/apps/api-gateway/src/issuance/issuance.module.ts b/apps/api-gateway/src/issuance/issuance.module.ts index 6859908df..e8ef29c9e 100644 --- a/apps/api-gateway/src/issuance/issuance.module.ts +++ b/apps/api-gateway/src/issuance/issuance.module.ts @@ -5,7 +5,7 @@ import { IssuanceService } from './issuance.service'; import { CommonService } from '@credebl/common'; import { HttpModule } from '@nestjs/axios'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CommonConstants } from '@credebl/common/common.constant'; import { NATSClient } from '@credebl/common/NATSClient'; @@ -25,6 +25,6 @@ import { NATSClient } from '@credebl/common/NATSClient'; ]) ], controllers: [IssuanceController], - providers: [IssuanceService, CommonService, AwsService, NATSClient] + providers: [IssuanceService, CommonService, StorageService, NATSClient] }) export class IssuanceModule {} diff --git a/apps/api-gateway/src/main.ts b/apps/api-gateway/src/main.ts index 67f0d0bb4..c696edeec 100755 --- a/apps/api-gateway/src/main.ts +++ b/apps/api-gateway/src/main.ts @@ -122,6 +122,7 @@ async function bootstrap(): Promise { app.use(express.static('invoice-pdf')); app.use(express.static('uploadedFiles/bulk-verification-templates')); app.use(express.static('uploadedFiles/import')); + app.use('/uploadedFiles', express.static('uploadedFiles')); // Use custom updatable global pipes const reflector = app.get(Reflector); app.useGlobalPipes(new UpdatableValidationPipe(reflector, { whitelist: true, transform: true })); diff --git a/apps/api-gateway/src/organization/organization.module.ts b/apps/api-gateway/src/organization/organization.module.ts index f49bff461..54d36a13b 100644 --- a/apps/api-gateway/src/organization/organization.module.ts +++ b/apps/api-gateway/src/organization/organization.module.ts @@ -7,7 +7,7 @@ import { Module } from '@nestjs/common'; import { OrganizationController } from './organization.controller'; import { OrganizationService } from './organization.service'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CommonConstants } from '@credebl/common/common.constant'; import { NATSClient } from '@credebl/common/NATSClient'; @Module({ @@ -28,6 +28,6 @@ import { NATSClient } from '@credebl/common/NATSClient'; ]) ], controllers: [OrganizationController], - providers: [OrganizationService, CommonService, AwsService, NATSClient] + providers: [OrganizationService, CommonService, StorageService, NATSClient] }) export class OrganizationModule {} diff --git a/apps/api-gateway/src/user/user.controller.ts b/apps/api-gateway/src/user/user.controller.ts index 24d4918d4..ac817300a 100644 --- a/apps/api-gateway/src/user/user.controller.ts +++ b/apps/api-gateway/src/user/user.controller.ts @@ -47,7 +47,7 @@ import { UpdatePlatformSettingsDto } from './dto/update-platform-settings.dto'; import { Roles } from '../authz/decorators/roles.decorator'; import { OrgRolesGuard } from '../authz/guards/org-roles.guard'; import { OrgRoles } from 'libs/org-roles/enums'; -import { AwsService } from '@credebl/aws/aws.service'; +import { StorageService } from '@credebl/storage'; import { PaginationDto } from '@credebl/common/dtos/pagination.dto'; import { UserAccessGuard } from '../authz/guards/user-access-guard'; import { TrimStringParamPipe } from '@credebl/common/cast.helper'; @@ -61,7 +61,7 @@ export class UserController { constructor( private readonly userService: UserService, private readonly commonService: CommonService, - private readonly awsService: AwsService + private readonly awsService: StorageService ) {} /** diff --git a/apps/api-gateway/src/user/user.module.ts b/apps/api-gateway/src/user/user.module.ts index 73f4d7de7..517ec3ba7 100644 --- a/apps/api-gateway/src/user/user.module.ts +++ b/apps/api-gateway/src/user/user.module.ts @@ -6,7 +6,7 @@ import { Module } from '@nestjs/common'; import { UserController } from './user.controller'; import { UserService } from './user.service'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CommonConstants } from '@credebl/common/common.constant'; import { NATSClient } from '@credebl/common/NATSClient'; @@ -27,6 +27,6 @@ import { NATSClient } from '@credebl/common/NATSClient'; ]) ], controllers: [UserController], - providers: [UserService, CommonService, AwsService, NATSClient] + providers: [UserService, CommonService, StorageService, NATSClient] }) export class UserModule {} diff --git a/apps/api-gateway/src/webhook/webhook.module.ts b/apps/api-gateway/src/webhook/webhook.module.ts index 3fcf9e7d8..088d697bc 100644 --- a/apps/api-gateway/src/webhook/webhook.module.ts +++ b/apps/api-gateway/src/webhook/webhook.module.ts @@ -5,7 +5,7 @@ import { WebhookService } from './webhook.service'; import { CommonService } from '@credebl/common'; import { HttpModule } from '@nestjs/axios'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CommonConstants } from '@credebl/common/common.constant'; import { NATSClient } from '@credebl/common/NATSClient'; @@ -25,6 +25,6 @@ import { NATSClient } from '@credebl/common/NATSClient'; ]) ], controllers: [WebhookController], - providers: [WebhookService, CommonService, AwsService, NATSClient] + providers: [WebhookService, CommonService, StorageService, NATSClient] }) export class WebhookModule {} diff --git a/apps/api-gateway/src/x509/x509.module.ts b/apps/api-gateway/src/x509/x509.module.ts index 992651c9a..e70384cbf 100644 --- a/apps/api-gateway/src/x509/x509.module.ts +++ b/apps/api-gateway/src/x509/x509.module.ts @@ -5,7 +5,7 @@ import { Module } from '@nestjs/common'; import { X509Controller } from './x509.controller'; import { X509Service } from './x509.service'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CommonConstants } from '@credebl/common/common.constant'; import { NATSClient } from '@credebl/common/NATSClient'; @Module({ @@ -25,6 +25,6 @@ import { NATSClient } from '@credebl/common/NATSClient'; ]) ], controllers: [X509Controller], - providers: [X509Service, AwsService, NATSClient] + providers: [X509Service, StorageService, NATSClient] }) export class X509Module {} diff --git a/apps/issuance/src/issuance.module.ts b/apps/issuance/src/issuance.module.ts index e5c0bc889..7686d1902 100644 --- a/apps/issuance/src/issuance.module.ts +++ b/apps/issuance/src/issuance.module.ts @@ -12,7 +12,7 @@ import { EmailDto } from '@credebl/common/dtos/email.dto'; import { BullModule } from '@nestjs/bull'; import { CacheModule } from '@nestjs/cache-manager'; import { BulkIssuanceProcessor } from './issuance.processor'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { UserActivityRepository } from 'libs/user-activity/repositories'; import { CommonConstants, MICRO_SERVICE_NAME } from '@credebl/common/common.constant'; import { LoggerModule } from '@credebl/logger/logger.module'; @@ -61,7 +61,7 @@ import { NATSClient } from '@credebl/common/NATSClient'; OutOfBandIssuance, EmailDto, BulkIssuanceProcessor, - AwsService, + StorageService, NATSClient, { provide: MICRO_SERVICE_NAME, diff --git a/apps/issuance/src/issuance.service.ts b/apps/issuance/src/issuance.service.ts index 04dce88af..8e5d5031c 100644 --- a/apps/issuance/src/issuance.service.ts +++ b/apps/issuance/src/issuance.service.ts @@ -66,7 +66,7 @@ import { convertUrlToDeepLinkUrl, getAgentUrl, paginator } from '@credebl/common import { InjectQueue } from '@nestjs/bull'; import { Queue } from 'bull'; import { FileUploadStatus, FileUploadType } from 'apps/api-gateway/src/enum'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { io } from 'socket.io-client'; import { IIssuedCredentialSearchParams, IssueCredentialType } from 'apps/api-gateway/src/issuance/interfaces'; import { @@ -107,7 +107,7 @@ export class IssuanceService { @Inject(CACHE_MANAGER) private readonly cacheManager: Cache, private readonly outOfBandIssuance: OutOfBandIssuance, private readonly emailData: EmailDto, - private readonly awsService: AwsService, + private readonly awsService: StorageService, @InjectQueue('bulk-issuance') private readonly bulkIssuanceQueue: Queue, // TODO: Remove duplicate, unused variable @Inject(CACHE_MANAGER) private readonly cacheService: Cache, @@ -1290,7 +1290,6 @@ export class IssuanceService { credentialPayload.credentialType = SchemaType.INDY; credentialPayload.schemaName = credentialDetails.schemaName; } - const getFileDetails = await this.awsService.getFile(importFileDetails.fileKey); const csvData: string = getFileDetails.Body.toString(); diff --git a/apps/organization/src/organization.module.ts b/apps/organization/src/organization.module.ts index ffbdd07a4..c1dba9f9e 100644 --- a/apps/organization/src/organization.module.ts +++ b/apps/organization/src/organization.module.ts @@ -18,7 +18,7 @@ import { getNatsOptions } from '@credebl/common/nats.config'; import { ClientRegistrationService } from '@credebl/client-registration'; import { KeycloakUrlService } from '@credebl/keycloak-url'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CommonConstants } from '@credebl/common/common.constant'; import { GlobalConfigModule } from '@credebl/config/global-config.module'; import { ConfigModule as PlatformConfig } from '@credebl/config/config.module'; @@ -61,7 +61,7 @@ import { NATSClient } from '@credebl/common/NATSClient'; UserActivityService, ClientRegistrationService, KeycloakUrlService, - AwsService, + StorageService, NATSClient ], exports: [OrganizationRepository] diff --git a/apps/organization/src/organization.service.ts b/apps/organization/src/organization.service.ts index 03e98d4bb..bf9960022 100755 --- a/apps/organization/src/organization.service.ts +++ b/apps/organization/src/organization.service.ts @@ -42,7 +42,7 @@ import { UserActivityService } from '@credebl/user-activity'; import { ClientRegistrationService } from '@credebl/client-registration/client-registration.service'; import { map } from 'rxjs/operators'; import { Cache } from 'cache-manager'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CACHE_MANAGER } from '@nestjs/cache-manager'; import { IOrgCredentials, @@ -77,7 +77,7 @@ export class OrganizationService { private readonly organizationRepository: OrganizationRepository, private readonly orgRoleService: OrgRolesService, private readonly userOrgRoleService: UserOrgRolesService, - private readonly awsService: AwsService, + private readonly awsService: StorageService, private readonly userActivityService: UserActivityService, private readonly logger: Logger, // TODO: Remove duplicate, unused variable @@ -498,7 +498,7 @@ export class OrganizationService { imgData, 'png', 'orgLogo', - process.env.AWS_ORG_LOGO_BUCKET_NAME, + process.env.ORG_LOGO_BUCKET_NAME, 'base64', 'orgLogos' ); diff --git a/apps/user/src/fido/fido.module.ts b/apps/user/src/fido/fido.module.ts index e3125189a..de737b4fb 100644 --- a/apps/user/src/fido/fido.module.ts +++ b/apps/user/src/fido/fido.module.ts @@ -19,7 +19,7 @@ import { UserOrgRolesRepository } from 'libs/user-org-roles/repositories'; import { UserOrgRolesService } from '@credebl/user-org-roles'; import { UserRepository } from '../../repositories/user.repository'; import { UserService } from '../user.service'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { NATSClient } from '@credebl/common/NATSClient'; @Module({ @@ -35,10 +35,10 @@ import { NATSClient } from '@credebl/common/NATSClient'; ]), HttpModule, CommonModule -], + ], controllers: [FidoController], providers: [ - AwsService, + StorageService, UserService, PrismaService, FidoService, @@ -56,6 +56,6 @@ import { NATSClient } from '@credebl/common/NATSClient'; UserActivityService, UserActivityRepository, NATSClient -] + ] }) -export class FidoModule { } +export class FidoModule {} diff --git a/apps/user/src/user.module.ts b/apps/user/src/user.module.ts index caf3fe38f..b9c2019ae 100644 --- a/apps/user/src/user.module.ts +++ b/apps/user/src/user.module.ts @@ -2,7 +2,7 @@ import { ClientsModule, Transport } from '@nestjs/microservices'; import { Logger, Module } from '@nestjs/common'; import { OrgRolesModule, OrgRolesService } from '@credebl/org-roles'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { ClientRegistrationService } from '@credebl/client-registration'; import { CommonConstants } from '@credebl/common/common.constant'; import { CommonModule } from '@credebl/common'; @@ -46,7 +46,7 @@ import { getNatsOptions } from '@credebl/common/nats.config'; ], controllers: [UserController], providers: [ - AwsService, + StorageService, UserService, UserRepository, PrismaService, diff --git a/apps/user/src/user.service.ts b/apps/user/src/user.service.ts index 59f46e429..71ee46e1e 100644 --- a/apps/user/src/user.service.ts +++ b/apps/user/src/user.service.ts @@ -53,7 +53,7 @@ import { v4 as uuidv4 } from 'uuid'; import { Invitation, ProviderType, SessionType, TokenType, UserRole } from '@credebl/enum/enum'; import validator from 'validator'; import { DISALLOWED_EMAIL_DOMAIN } from '@credebl/common/common.constant'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { IUsersActivity } from 'libs/user-activity/interface'; import { ISendVerificationEmail, @@ -83,7 +83,7 @@ export class UserService { private readonly userOrgRoleService: UserOrgRolesService, private readonly userActivityService: UserActivityService, private readonly userRepository: UserRepository, - private readonly awsService: AwsService, + private readonly awsService: StorageService, private readonly userDevicesRepository: UserDevicesRepository, private readonly logger: Logger, @Inject('NATS_CLIENT') private readonly userServiceProxy: ClientProxy, diff --git a/apps/utility/src/utilities.module.ts b/apps/utility/src/utilities.module.ts index 15418c55f..2a030f932 100644 --- a/apps/utility/src/utilities.module.ts +++ b/apps/utility/src/utilities.module.ts @@ -7,7 +7,7 @@ import { PrismaService } from '@credebl/prisma-service'; import { UtilitiesController } from './utilities.controller'; import { UtilitiesService } from './utilities.service'; import { UtilitiesRepository } from './utilities.repository'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { CommonConstants } from '@credebl/common/common.constant'; import { GlobalConfigModule } from '@credebl/config/global-config.module'; import { ConfigModule as PlatformConfig } from '@credebl/config/config.module'; @@ -35,6 +35,6 @@ import { ContextInterceptorModule } from '@credebl/context/contextInterceptorMod CacheModule.register() ], controllers: [UtilitiesController], - providers: [UtilitiesService, Logger, PrismaService, UtilitiesRepository, AwsService] + providers: [UtilitiesService, Logger, PrismaService, UtilitiesRepository, StorageService] }) export class UtilitiesModule {} diff --git a/apps/utility/src/utilities.service.ts b/apps/utility/src/utilities.service.ts index 89145b43b..bde5fe752 100644 --- a/apps/utility/src/utilities.service.ts +++ b/apps/utility/src/utilities.service.ts @@ -1,6 +1,6 @@ import { Injectable, NotFoundException } from '@nestjs/common'; -import { AwsService } from '@credebl/aws'; +import { StorageService } from '@credebl/storage'; import { BaseService } from 'libs/service/base.service'; import { EmailDto } from '@credebl/common/dtos/email.dto'; import { EmailService } from '@credebl/common/email.service'; @@ -17,7 +17,7 @@ export class UtilitiesService extends BaseService { constructor( private readonly utilitiesRepository: UtilitiesRepository, - private readonly awsService: AwsService, + private readonly awsService: StorageService, private readonly emailService: EmailService ) { super('UtilitiesService'); @@ -71,6 +71,10 @@ export class UtilitiesService extends BaseService { uuid, payload.storeObj ); + const storageType = process.env.FILE_STORAGE_TYPE || 'aws'; + if ('local' === storageType || 'rustfs' === storageType) { + return uploadResult.Location; + } const url: string = `${process.env.SHORTENED_URL_DOMAIN}/${uploadResult.Key}`; return url; } catch (error) { diff --git a/libs/aws/src/aws.module.ts b/libs/aws/src/aws.module.ts deleted file mode 100644 index 1a2a90f39..000000000 --- a/libs/aws/src/aws.module.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Module } from '@nestjs/common'; -import { AwsService } from './aws.service'; - -@Module({ - providers: [AwsService], - exports: [AwsService] -}) -export class AwsModule {} diff --git a/libs/aws/src/aws.service.spec.ts b/libs/aws/src/aws.service.spec.ts deleted file mode 100644 index f37dab349..000000000 --- a/libs/aws/src/aws.service.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { AwsService } from './aws.service'; - -describe('AwsService', () => { - let service: AwsService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [AwsService] - }).compile(); - - service = module.get(AwsService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/libs/aws/src/index.ts b/libs/aws/src/index.ts deleted file mode 100644 index 182a99dc1..000000000 --- a/libs/aws/src/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './aws.module'; -export * from './aws.service'; diff --git a/libs/aws/package.json b/libs/storage/package.json similarity index 86% rename from libs/aws/package.json rename to libs/storage/package.json index 56590cbf5..4e7654496 100644 --- a/libs/aws/package.json +++ b/libs/storage/package.json @@ -1,5 +1,5 @@ { - "name": "@credebl/aws", + "name": "@credebl/storage", "main": "src/index", "types": "src/index", "version": "0.0.1", @@ -8,7 +8,7 @@ ], "scripts": { "build": "pnpm run clean && pnpm run compile", - "clean": "rimraf ../../dist/libs/aws", + "clean": "rimraf ../../dist/libs/storage", "compile": "tsc -p tsconfig.build.json", "test": "jest" }, diff --git a/libs/storage/src/index.ts b/libs/storage/src/index.ts new file mode 100644 index 000000000..0d903c450 --- /dev/null +++ b/libs/storage/src/index.ts @@ -0,0 +1,4 @@ +export * from './storage.module'; +export * from './storage.service'; +export * from './storage.interface'; +export * from './providers'; diff --git a/libs/aws/src/aws.service.ts b/libs/storage/src/providers/base-s3-storage.provider.ts similarity index 64% rename from libs/aws/src/aws.service.ts rename to libs/storage/src/providers/base-s3-storage.provider.ts index 19286edea..4610b9008 100644 --- a/libs/aws/src/aws.service.ts +++ b/libs/storage/src/providers/base-s3-storage.provider.ts @@ -1,34 +1,27 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus } from '@nestjs/common'; + +import { IStorageService } from '../storage.interface'; import { RpcException } from '@nestjs/microservices'; import { S3 } from 'aws-sdk'; import { promisify } from 'util'; -@Injectable() -export class AwsService { - private s3: S3; - private s4: S3; - private s3StoreObject: S3; - - constructor() { - this.s3 = new S3({ - accessKeyId: process.env.AWS_ACCESS_KEY, - secretAccessKey: process.env.AWS_SECRET_KEY, - region: process.env.AWS_REGION - }); +export abstract class BaseS3StorageService implements IStorageService { + protected s3: S3; + protected s4: S3; + protected s3StoreObject: S3; - this.s4 = new S3({ - accessKeyId: process.env.AWS_PUBLIC_ACCESS_KEY, - secretAccessKey: process.env.AWS_PUBLIC_SECRET_KEY, - region: process.env.AWS_PUBLIC_REGION - }); - - this.s3StoreObject = new S3({ - accessKeyId: process.env.AWS_S3_STOREOBJECT_ACCESS_KEY, - secretAccessKey: process.env.AWS_S3_STOREOBJECT_SECRET_KEY, - region: process.env.AWS_S3_STOREOBJECT_REGION - }); + constructor( + s3Config: Record, + s4Config: Record, + storeObjectConfig: Record + ) { + this.s3 = new S3(s3Config); + this.s4 = new S3(s4Config); + this.s3StoreObject = new S3(storeObjectConfig); } + abstract getPublicUrl(bucketName: string, fileKey: string): string; + async uploadFileToS3Bucket( fileBuffer: Buffer, ext: string, @@ -39,7 +32,7 @@ export class AwsService { ): Promise { const timestamp = Date.now(); const putObjectAsync = promisify(this.s4.putObject).bind(this.s4); - + const fileKey = `${pathAWS}/${encodeURIComponent(filename)}-${timestamp}.${ext}`; try { await putObjectAsync({ Bucket: `${bucketName}`, @@ -48,9 +41,7 @@ export class AwsService { ContentEncoding: encoding, ContentType: `image/png` }); - - const imageUrl = `https://${bucketName}.s3.${process.env.AWS_PUBLIC_REGION}.amazonaws.com/${pathAWS}/${encodeURIComponent(filename)}-${timestamp}.${ext}`; - return imageUrl; + return this.getPublicUrl(bucketName, fileKey); } catch (error) { throw new HttpException(error, HttpStatus.SERVICE_UNAVAILABLE); } @@ -58,11 +49,10 @@ export class AwsService { async uploadCsvFile(key: string, body: unknown): Promise { const params: AWS.S3.PutObjectRequest = { - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.FILE_SHARING_BUCKET, Key: key, Body: 'string' === typeof body ? body : body.toString() }; - try { await this.s3.upload(params).promise(); } catch (error) { @@ -72,7 +62,7 @@ export class AwsService { async getFile(key: string): Promise { const params: AWS.S3.GetObjectRequest = { - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.FILE_SHARING_BUCKET, Key: key }; try { @@ -84,7 +74,7 @@ export class AwsService { async deleteFile(key: string): Promise { const params: AWS.S3.DeleteObjectRequest = { - Bucket: process.env.AWS_BUCKET, + Bucket: process.env.FILE_SHARING_BUCKET, Key: key }; try { @@ -98,7 +88,7 @@ export class AwsService { const objKey: string = persistent.valueOf() ? `persist/${key}` : `default/${key}`; const buf = Buffer.from(JSON.stringify(body)); const params: AWS.S3.PutObjectRequest = { - Bucket: process.env.AWS_S3_STOREOBJECT_BUCKET, + Bucket: process.env.STOREOBJECT_BUCKET, Body: buf, Key: objKey, ContentEncoding: 'base64', @@ -106,8 +96,7 @@ export class AwsService { }; try { - const receivedData = await this.s3StoreObject.upload(params).promise(); - return receivedData; + return await this.s3StoreObject.upload(params).promise(); } catch (error) { throw new RpcException(error.response ? error.response : error); } diff --git a/libs/storage/src/providers/index.ts b/libs/storage/src/providers/index.ts new file mode 100644 index 000000000..30da51f52 --- /dev/null +++ b/libs/storage/src/providers/index.ts @@ -0,0 +1,3 @@ +export * from './s3-storage.provider'; +export * from './rustfs-storage.provider'; +export * from './local-fs-storage.provider'; diff --git a/libs/storage/src/providers/local-fs-storage.provider.ts b/libs/storage/src/providers/local-fs-storage.provider.ts new file mode 100644 index 000000000..f4d633192 --- /dev/null +++ b/libs/storage/src/providers/local-fs-storage.provider.ts @@ -0,0 +1,85 @@ +import * as path from 'path'; + +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; + +import { IStorageService } from '../storage.interface'; +import { S3 } from 'aws-sdk'; +import { promises as fs } from 'fs'; + +@Injectable() +export class LocalFsStorageService implements IStorageService { + private localStoragePath: string; + + constructor() { + this.localStoragePath = path.resolve(process.cwd(), 'uploadedFiles'); + } + + private async ensureDir(dirPath: string): Promise { + await fs.mkdir(dirPath, { recursive: true }); + } + + private resolveUnderBase(baseDir: string, relativePath: string): string { + const resolvedBase = path.resolve(baseDir); + const resolvedTarget = path.resolve(baseDir, relativePath); + if (!resolvedTarget.startsWith(`${resolvedBase}${path.sep}`) && resolvedTarget !== resolvedBase) { + throw new HttpException('Invalid file key/path', HttpStatus.BAD_REQUEST); + } + return resolvedTarget; + } + + async uploadFileToS3Bucket( + fileBuffer: Buffer, + ext: string, + filename: string, + bucketName: string, + encoding: string, + pathAWS: string = '' + ): Promise { + const bucketDir = path.join(this.localStoragePath, bucketName); + await this.ensureDir(path.join(bucketDir, pathAWS)); + const timestamp = Date.now(); + const fileKey = `${pathAWS}/${encodeURIComponent(filename)}-${timestamp}.${ext}`; + const filePath = this.resolveUnderBase(bucketDir, fileKey); + await fs.writeFile(filePath, fileBuffer); + return `${process.env.PLATFORM_URL}/uploadedFiles/${bucketName}/${fileKey}`; + } + + async uploadCsvFile(key: string, body: unknown): Promise { + const bucketDir = path.join(this.localStoragePath, process.env.FILE_SHARING_BUCKET); + const filePath = path.join(bucketDir, key); + await this.ensureDir(path.dirname(filePath)); + const data = 'string' === typeof body ? body : body.toString(); + await fs.writeFile(filePath, data); + } + + async getFile(key: string): Promise { + const filePath = path.join(this.localStoragePath, process.env.FILE_SHARING_BUCKET, key); + const fileContent = await fs.readFile(filePath); + return { Body: fileContent } as AWS.S3.GetObjectOutput; + } + + async deleteFile(key: string): Promise { + const filePath = path.join(this.localStoragePath, process.env.FILE_SHARING_BUCKET, key); + await fs.unlink(filePath); + } + + async storeObject(persistent: boolean, key: string, body: unknown): Promise { + const objKey = persistent ? `persist/${key}` : `default/${key}`; + const objFilePath = `${objKey}.json`; + const bucketDir = path.join(this.localStoragePath, process.env.STOREOBJECT_BUCKET); + const filePath = path.join(bucketDir, objFilePath); + const publicUrl = `${process.env.PLATFORM_URL}/uploadedFiles/${process.env.STOREOBJECT_BUCKET}/${objFilePath}`; + await this.ensureDir(path.dirname(filePath)); + const buf = Buffer.from(JSON.stringify(body)); + await fs.writeFile(filePath, buf); + return { + Expiration: 'expiry-date="Sun, 05 Jul 2026 00:00:00 GMT", rule-id="Default_folder"', + ETag: '', + ServerSideEncryption: '', + Location: publicUrl, + key: objFilePath, + Key: objFilePath, + Bucket: process.env.STOREOBJECT_BUCKET + } as S3.ManagedUpload.SendData; + } +} diff --git a/libs/storage/src/providers/rustfs-storage.provider.ts b/libs/storage/src/providers/rustfs-storage.provider.ts new file mode 100644 index 000000000..c90287b95 --- /dev/null +++ b/libs/storage/src/providers/rustfs-storage.provider.ts @@ -0,0 +1,27 @@ +import { BaseS3StorageService } from './base-s3-storage.provider'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class RustFsStorageService extends BaseS3StorageService { + constructor() { + const endpoint = process.env.RUSTFS_ENDPOINT || 'http://localhost:9000'; + + const baseConfig = { + accessKeyId: process.env.RUSTFS_ACCESS_KEY_ID, + secretAccessKey: process.env.RUSTFS_SECRET_ACCESS_KEY, + endpoint, + s3ForcePathStyle: true + }; + + const s3Config = { ...baseConfig, region: process.env.AWS_REGION }; + const s4Config = { ...baseConfig, region: process.env.AWS_PUBLIC_REGION }; + const storeObjectConfig = { ...baseConfig, region: process.env.AWS_S3_STOREOBJECT_REGION }; + + super(s3Config, s4Config, storeObjectConfig); + } + + getPublicUrl(bucketName: string, fileKey: string): string { + const endpoint = process.env.RUSTFS_ENDPOINT || 'http://localhost:9000'; + return `${endpoint}/${bucketName}/${fileKey}`; + } +} diff --git a/libs/storage/src/providers/s3-storage.provider.ts b/libs/storage/src/providers/s3-storage.provider.ts new file mode 100644 index 000000000..546c82bdc --- /dev/null +++ b/libs/storage/src/providers/s3-storage.provider.ts @@ -0,0 +1,31 @@ +import { BaseS3StorageService } from './base-s3-storage.provider'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class S3StorageService extends BaseS3StorageService { + constructor() { + const s3Config = { + accessKeyId: process.env.AWS_ACCESS_KEY, + secretAccessKey: process.env.AWS_SECRET_KEY, + region: process.env.AWS_REGION + }; + + const s4Config = { + accessKeyId: process.env.AWS_PUBLIC_ACCESS_KEY, + secretAccessKey: process.env.AWS_PUBLIC_SECRET_KEY, + region: process.env.AWS_PUBLIC_REGION + }; + + const storeObjectConfig = { + accessKeyId: process.env.AWS_S3_STOREOBJECT_ACCESS_KEY, + secretAccessKey: process.env.AWS_S3_STOREOBJECT_SECRET_KEY, + region: process.env.AWS_S3_STOREOBJECT_REGION + }; + + super(s3Config, s4Config, storeObjectConfig); + } + + getPublicUrl(bucketName: string, fileKey: string): string { + return `https://${bucketName}.s3.${process.env.AWS_PUBLIC_REGION}.amazonaws.com/${fileKey}`; + } +} diff --git a/libs/storage/src/storage.interface.ts b/libs/storage/src/storage.interface.ts new file mode 100644 index 000000000..b212ed921 --- /dev/null +++ b/libs/storage/src/storage.interface.ts @@ -0,0 +1,20 @@ +import { S3 } from 'aws-sdk'; + +export interface IStorageService { + uploadFileToS3Bucket( + fileBuffer: Buffer, + ext: string, + filename: string, + bucketName: string, + encoding: string, + pathAWS?: string + ): Promise; + + uploadCsvFile(key: string, body: unknown): Promise; + + getFile(key: string): Promise; + + deleteFile(key: string): Promise; + + storeObject(persistent: boolean, key: string, body: unknown): Promise; +} diff --git a/libs/storage/src/storage.module.ts b/libs/storage/src/storage.module.ts new file mode 100644 index 000000000..a8a09c64c --- /dev/null +++ b/libs/storage/src/storage.module.ts @@ -0,0 +1,11 @@ +import { LocalFsStorageService } from './providers/local-fs-storage.provider'; +import { Module } from '@nestjs/common'; +import { RustFsStorageService } from './providers/rustfs-storage.provider'; +import { S3StorageService } from './providers/s3-storage.provider'; +import { StorageService } from './storage.service'; + +@Module({ + providers: [StorageService, S3StorageService, RustFsStorageService, LocalFsStorageService], + exports: [StorageService] +}) +export class StorageModule {} diff --git a/libs/storage/src/storage.service.ts b/libs/storage/src/storage.service.ts new file mode 100644 index 000000000..ac91ca0f4 --- /dev/null +++ b/libs/storage/src/storage.service.ts @@ -0,0 +1,53 @@ +import { IStorageService } from './storage.interface'; +import { Injectable } from '@nestjs/common'; +import { LocalFsStorageService } from './providers/local-fs-storage.provider'; +import { RustFsStorageService } from './providers/rustfs-storage.provider'; +import { S3 } from 'aws-sdk'; +import { S3StorageService } from './providers/s3-storage.provider'; + +@Injectable() +export class StorageService implements IStorageService { + private storage: IStorageService; + + constructor() { + const storageType = process.env.FILE_STORAGE_TYPE || 'aws'; + switch (storageType) { + case 'local': + this.storage = new LocalFsStorageService(); + break; + case 'rustfs': + this.storage = new RustFsStorageService(); + break; + default: + this.storage = new S3StorageService(); + break; + } + } + + async uploadFileToS3Bucket( + fileBuffer: Buffer, + ext: string, + filename: string, + bucketName: string, + encoding: string, + pathAWS?: string + ): Promise { + return this.storage.uploadFileToS3Bucket(fileBuffer, ext, filename, bucketName, encoding, pathAWS); + } + + async uploadCsvFile(key: string, body: unknown): Promise { + return this.storage.uploadCsvFile(key, body); + } + + async getFile(key: string): Promise { + return this.storage.getFile(key); + } + + async deleteFile(key: string): Promise { + return this.storage.deleteFile(key); + } + + async storeObject(persistent: boolean, key: string, body: unknown): Promise { + return this.storage.storeObject(persistent, key, body); + } +} diff --git a/libs/aws/tsconfig.build.json b/libs/storage/tsconfig.build.json similarity index 71% rename from libs/aws/tsconfig.build.json rename to libs/storage/tsconfig.build.json index 74c2e991f..6df4ef389 100644 --- a/libs/aws/tsconfig.build.json +++ b/libs/storage/tsconfig.build.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.build.json", "compilerOptions": { - "outDir": "../../dist/libs/aws" + "outDir": "../../dist/libs/storage" }, "include": ["src/**/*"] } diff --git a/libs/aws/tsconfig.json b/libs/storage/tsconfig.json similarity index 100% rename from libs/aws/tsconfig.json rename to libs/storage/tsconfig.json diff --git a/nest-cli.json b/nest-cli.json index 5238a9b41..2b526e524 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -35,15 +35,6 @@ "tsConfigPath": "apps/api-gateway/tsconfig.app.json" } }, - "aws": { - "type": "library", - "root": "libs/aws", - "entryFile": "index", - "sourceRoot": "libs/aws/src", - "compilerOptions": { - "tsConfigPath": "libs/aws/tsconfig.lib.json" - } - }, "client-registration": { "type": "library", "root": "libs/client-registration", diff --git a/package.json b/package.json index c6bdb9daf..79f2dbe33 100644 --- a/package.json +++ b/package.json @@ -201,7 +201,7 @@ "^@credebl/user-org-roles(|/.*)$": "/libs/user-org-roles/src/$1", "^y/user-activity(|/.*)$": "/libs/user-activity/src/$1", "^@app/supabase(|/.*)$": "/libs/supabase/src/$1", - "^@credebl/aws(|/.*)$": "/libs/aws/src/$1", + "^credebl/utility(|/.*)$": "/libs/utility/src/$1", "^@credebl/config(|/.*)$": "/libs/config/src/$1", "^@credebl/context(|/.*)$": "/libs/context/src/$1", diff --git a/tsconfig.json b/tsconfig.json index 12be5645c..cfcd0e973 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,12 +2,6 @@ "extends": "./tsconfig.build.json", "compilerOptions": { "paths": { - "@credebl/aws": [ - "libs/aws/src" - ], - "@credebl/aws/*": [ - "libs/aws/src/*" - ], "@credebl/client-registration": [ "libs/client-registration/src" ], @@ -32,6 +26,12 @@ "@credebl/context/*": [ "libs/context/src/*" ], + "@credebl/storage": [ + "libs/storage/src" + ], + "@credebl/storage/*": [ + "libs/storage/src/*" + ], "@credebl/enum": [ "libs/enum/src" ],