feat: add rust fs and local storage support

.env.demo

@@ -109,24 +109,37 @@ DATABASE_URL="postgresql://postgres:postgres@your-ip:5432/credebl"
 AWS_ACCESS_KEY=
 AWS_SECRET_KEY=
 AWS_REGION=
-AWS_BUCKET=
+FILE_SHARING_BUCKET=
 
 # Used for Adding org-logo during org creation and update (Optional- Can be skipped if no image is added during org creation and updation)
 # Provide aws bucket access credentails with write access to the bucket as logo upload is done using signed url which requires putObject permission.
 AWS_PUBLIC_ACCESS_KEY=
 AWS_PUBLIC_SECRET_KEY=
 AWS_PUBLIC_REGION=
-AWS_ORG_LOGO_BUCKET_NAME=
+ORG_LOGO_BUCKET_NAME=
 
 # Used for storing connection URL generated from Agent and creating shortened URL (As connecting to org requires Shortened url)
 # Provide aws bucket access credentails with write access to the bucket as storing shortened url is done using putObject operation.
 AWS_S3_STOREOBJECT_ACCESS_KEY=
 AWS_S3_STOREOBJECT_SECRET_KEY=
 AWS_S3_STOREOBJECT_REGION=
-AWS_S3_STOREOBJECT_BUCKET=
+STOREOBJECT_BUCKET=
 
 # Provide bucket used for storing shortened url objects e.g. 'https://s3.region-code.amazonaws.com/bucket-name'
-SHORTENED_URL_DOMAIN='https://s3.AWS_S3_STOREOBJECT_REGION.amazonaws.com/AWS_S3_STOREOBJECT_BUCKET'
+SHORTENED_URL_DOMAIN='https://s3.AWS_S3_STOREOBJECT_REGION.amazonaws.com/STOREOBJECT_BUCKET'
+
+# Storage type: 'aws' (default), 'rustfs' (S3-compatible), or 'local' (filesystem)
+# When set to 'local', AWS/RustFS credentials are not required
+FILE_STORAGE_TYPE=aws
+
+RUSTFS_ENDPOINT=
+RUSTFS_ACCESS_KEY_ID=
+RUSTFS_SECRET_ACCESS_KEY=
+RUSTFS_REGION=
+# make sure to add the bucket names in the below variables if you are using local rustfs for file storage, as these buckets need to be created in rustfs and the same bucket names should be used in the code for storing and retrieving files.
+# FILE_SHARING_BUCKET=
+# ORG_LOGO_BUCKET_NAME=
+# STOREOBJECT_BUCKET=
 
 # Specify your domain/subdomain responsible for deeplinking with 'url' as a query param e.g. 'https://your-deeplink-domain?url='
 DEEPLINK_DOMAIN='https://link.credebl.id?url='

.gitignore

@@ -5,6 +5,7 @@ sonar-project.properties
 .scannerwork/*
 coverage
 libs/prisma-service/prisma/data/credebl-master-table/credebl-master-table.json
+uploadedFiles/*
 uploadedFles/exports
 uploadedFles/import
 uploadedFles/export

apps/api-gateway/src/issuance/issuance.controller.ts

@@ -69,7 +69,7 @@ import {
   IssueCredentialType,
   UploadedFileDetails
 } from './interfaces';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { FileInterceptor } from '@nestjs/platform-express';
 import { v4 as uuidv4 } from 'uuid';
 import { RpcException } from '@nestjs/microservices';
@@ -90,7 +90,7 @@ import { IWebhookUrlInfo } from '@credebl/common/interfaces/webhook.interface';
 export class IssuanceController {
   constructor(
     private readonly issueCredentialService: IssuanceService,
-    private readonly awsService: AwsService
+    private readonly awsService: StorageService
   ) {}
   private readonly logger = new Logger('IssuanceController');
 

apps/api-gateway/src/issuance/issuance.module.ts

@@ -5,7 +5,7 @@ import { IssuanceService } from './issuance.service';
 import { CommonService } from '@credebl/common';
 import { HttpModule } from '@nestjs/axios';
 import { getNatsOptions } from '@credebl/common/nats.config';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { NATSClient } from '@credebl/common/NATSClient';
 
@@ -25,6 +25,6 @@ import { NATSClient } from '@credebl/common/NATSClient';
     ])
   ],
   controllers: [IssuanceController],
-  providers: [IssuanceService, CommonService, AwsService, NATSClient]
+  providers: [IssuanceService, CommonService, StorageService, NATSClient]
 })
 export class IssuanceModule {}

apps/api-gateway/src/main.ts

@@ -122,6 +122,7 @@ async function bootstrap(): Promise<void> {
   app.use(express.static('invoice-pdf'));
   app.use(express.static('uploadedFiles/bulk-verification-templates'));
   app.use(express.static('uploadedFiles/import'));
+  app.use('/uploadedFiles', express.static('uploadedFiles'));
   // Use custom updatable global pipes
   const reflector = app.get(Reflector);
   app.useGlobalPipes(new UpdatableValidationPipe(reflector, { whitelist: true, transform: true }));

apps/api-gateway/src/organization/organization.module.ts

@@ -7,7 +7,7 @@ import { Module } from '@nestjs/common';
 import { OrganizationController } from './organization.controller';
 import { OrganizationService } from './organization.service';
 import { getNatsOptions } from '@credebl/common/nats.config';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { NATSClient } from '@credebl/common/NATSClient';
 @Module({
@@ -28,6 +28,6 @@ import { NATSClient } from '@credebl/common/NATSClient';
     ])
   ],
   controllers: [OrganizationController],
-  providers: [OrganizationService, CommonService, AwsService, NATSClient]
+  providers: [OrganizationService, CommonService, StorageService, NATSClient]
 })
 export class OrganizationModule {}

apps/api-gateway/src/user/user.controller.ts

@@ -47,7 +47,7 @@ import { UpdatePlatformSettingsDto } from './dto/update-platform-settings.dto';
 import { Roles } from '../authz/decorators/roles.decorator';
 import { OrgRolesGuard } from '../authz/guards/org-roles.guard';
 import { OrgRoles } from 'libs/org-roles/enums';
-import { AwsService } from '@credebl/aws/aws.service';
+import { StorageService } from '@credebl/storage';
 import { PaginationDto } from '@credebl/common/dtos/pagination.dto';
 import { UserAccessGuard } from '../authz/guards/user-access-guard';
 import { TrimStringParamPipe } from '@credebl/common/cast.helper';
@@ -61,7 +61,7 @@ export class UserController {
   constructor(
     private readonly userService: UserService,
     private readonly commonService: CommonService,
-    private readonly awsService: AwsService
+    private readonly awsService: StorageService
   ) {}
 
   /**

apps/api-gateway/src/user/user.module.ts

@@ -6,7 +6,7 @@ import { Module } from '@nestjs/common';
 import { UserController } from './user.controller';
 import { UserService } from './user.service';
 import { getNatsOptions } from '@credebl/common/nats.config';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { NATSClient } from '@credebl/common/NATSClient';
 
@@ -27,6 +27,6 @@ import { NATSClient } from '@credebl/common/NATSClient';
     ])
   ],
   controllers: [UserController],
-  providers: [UserService, CommonService, AwsService, NATSClient]
+  providers: [UserService, CommonService, StorageService, NATSClient]
 })
 export class UserModule {}

apps/api-gateway/src/webhook/webhook.module.ts

@@ -5,7 +5,7 @@ import { WebhookService } from './webhook.service';
 import { CommonService } from '@credebl/common';
 import { HttpModule } from '@nestjs/axios';
 import { getNatsOptions } from '@credebl/common/nats.config';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { NATSClient } from '@credebl/common/NATSClient';
 
@@ -25,6 +25,6 @@ import { NATSClient } from '@credebl/common/NATSClient';
     ])
   ],
   controllers: [WebhookController],
-  providers: [WebhookService, CommonService, AwsService, NATSClient]
+  providers: [WebhookService, CommonService, StorageService, NATSClient]
 })
 export class WebhookModule {}

apps/api-gateway/src/x509/x509.module.ts

@@ -5,7 +5,7 @@ import { Module } from '@nestjs/common';
 import { X509Controller } from './x509.controller';
 import { X509Service } from './x509.service';
 import { getNatsOptions } from '@credebl/common/nats.config';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { NATSClient } from '@credebl/common/NATSClient';
 @Module({
@@ -25,6 +25,6 @@ import { NATSClient } from '@credebl/common/NATSClient';
     ])
   ],
   controllers: [X509Controller],
-  providers: [X509Service, AwsService, NATSClient]
+  providers: [X509Service, StorageService, NATSClient]
 })
 export class X509Module {}

apps/issuance/src/issuance.module.ts

@@ -12,7 +12,7 @@ import { EmailDto } from '@credebl/common/dtos/email.dto';
 import { BullModule } from '@nestjs/bull';
 import { CacheModule } from '@nestjs/cache-manager';
 import { BulkIssuanceProcessor } from './issuance.processor';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { UserActivityRepository } from 'libs/user-activity/repositories';
 import { CommonConstants, MICRO_SERVICE_NAME } from '@credebl/common/common.constant';
 import { LoggerModule } from '@credebl/logger/logger.module';
@@ -61,7 +61,7 @@ import { NATSClient } from '@credebl/common/NATSClient';
     OutOfBandIssuance,
     EmailDto,
     BulkIssuanceProcessor,
-    AwsService,
+    StorageService,
     NATSClient,
     {
       provide: MICRO_SERVICE_NAME,

apps/issuance/src/issuance.service.ts

@@ -66,7 +66,7 @@ import { convertUrlToDeepLinkUrl, getAgentUrl, paginator } from '@credebl/common
 import { InjectQueue } from '@nestjs/bull';
 import { Queue } from 'bull';
 import { FileUploadStatus, FileUploadType } from 'apps/api-gateway/src/enum';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { io } from 'socket.io-client';
 import { IIssuedCredentialSearchParams, IssueCredentialType } from 'apps/api-gateway/src/issuance/interfaces';
 import {
@@ -107,7 +107,7 @@ export class IssuanceService {
     @Inject(CACHE_MANAGER) private readonly cacheManager: Cache,
     private readonly outOfBandIssuance: OutOfBandIssuance,
     private readonly emailData: EmailDto,
-    private readonly awsService: AwsService,
+    private readonly awsService: StorageService,
     @InjectQueue('bulk-issuance') private readonly bulkIssuanceQueue: Queue,
     // TODO: Remove duplicate, unused variable
     @Inject(CACHE_MANAGER) private readonly cacheService: Cache,
@@ -1290,7 +1290,6 @@ export class IssuanceService {
         credentialPayload.credentialType = SchemaType.INDY;
         credentialPayload.schemaName = credentialDetails.schemaName;
       }
-
       const getFileDetails = await this.awsService.getFile(importFileDetails.fileKey);
       const csvData: string = getFileDetails.Body.toString();
 

apps/organization/src/organization.module.ts

@@ -18,7 +18,7 @@ import { getNatsOptions } from '@credebl/common/nats.config';
 import { ClientRegistrationService } from '@credebl/client-registration';
 import { KeycloakUrlService } from '@credebl/keycloak-url';
 
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { GlobalConfigModule } from '@credebl/config/global-config.module';
 import { ConfigModule as PlatformConfig } from '@credebl/config/config.module';
@@ -61,7 +61,7 @@ import { NATSClient } from '@credebl/common/NATSClient';
     UserActivityService,
     ClientRegistrationService,
     KeycloakUrlService,
-    AwsService,
+    StorageService,
     NATSClient
   ],
   exports: [OrganizationRepository]

apps/organization/src/organization.service.ts

@@ -42,7 +42,7 @@ import { UserActivityService } from '@credebl/user-activity';
 import { ClientRegistrationService } from '@credebl/client-registration/client-registration.service';
 import { map } from 'rxjs/operators';
 import { Cache } from 'cache-manager';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CACHE_MANAGER } from '@nestjs/cache-manager';
 import {
   IOrgCredentials,
@@ -77,7 +77,7 @@ export class OrganizationService {
     private readonly organizationRepository: OrganizationRepository,
     private readonly orgRoleService: OrgRolesService,
     private readonly userOrgRoleService: UserOrgRolesService,
-    private readonly awsService: AwsService,
+    private readonly awsService: StorageService,
     private readonly userActivityService: UserActivityService,
     private readonly logger: Logger,
     // TODO: Remove duplicate, unused variable
@@ -498,7 +498,7 @@ export class OrganizationService {
         imgData,
         'png',
         'orgLogo',
-        process.env.AWS_ORG_LOGO_BUCKET_NAME,
+        process.env.ORG_LOGO_BUCKET_NAME,
         'base64',
         'orgLogos'
       );

apps/user/src/fido/fido.module.ts

@@ -19,7 +19,7 @@ import { UserOrgRolesRepository } from 'libs/user-org-roles/repositories';
 import { UserOrgRolesService } from '@credebl/user-org-roles';
 import { UserRepository } from '../../repositories/user.repository';
 import { UserService } from '../user.service';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { NATSClient } from '@credebl/common/NATSClient';
 
 @Module({
@@ -35,10 +35,10 @@ import { NATSClient } from '@credebl/common/NATSClient';
     ]),
     HttpModule,
     CommonModule
-],
+  ],
   controllers: [FidoController],
   providers: [
-    AwsService,
+    StorageService,
     UserService,
     PrismaService,
     FidoService,
@@ -56,6 +56,6 @@ import { NATSClient } from '@credebl/common/NATSClient';
     UserActivityService,
     UserActivityRepository,
     NATSClient
-]
+  ]
 })
-export class FidoModule { }
+export class FidoModule {}

apps/user/src/user.module.ts

@@ -2,7 +2,7 @@ import { ClientsModule, Transport } from '@nestjs/microservices';
 import { Logger, Module } from '@nestjs/common';
 import { OrgRolesModule, OrgRolesService } from '@credebl/org-roles';
 
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { ClientRegistrationService } from '@credebl/client-registration';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { CommonModule } from '@credebl/common';
@@ -46,7 +46,7 @@ import { getNatsOptions } from '@credebl/common/nats.config';
   ],
   controllers: [UserController],
   providers: [
-    AwsService,
+    StorageService,
     UserService,
     UserRepository,
     PrismaService,

apps/user/src/user.service.ts

@@ -53,7 +53,7 @@ import { v4 as uuidv4 } from 'uuid';
 import { Invitation, ProviderType, SessionType, TokenType, UserRole } from '@credebl/enum/enum';
 import validator from 'validator';
 import { DISALLOWED_EMAIL_DOMAIN } from '@credebl/common/common.constant';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { IUsersActivity } from 'libs/user-activity/interface';
 import {
   ISendVerificationEmail,
@@ -83,7 +83,7 @@ export class UserService {
     private readonly userOrgRoleService: UserOrgRolesService,
     private readonly userActivityService: UserActivityService,
     private readonly userRepository: UserRepository,
-    private readonly awsService: AwsService,
+    private readonly awsService: StorageService,
     private readonly userDevicesRepository: UserDevicesRepository,
     private readonly logger: Logger,
     @Inject('NATS_CLIENT') private readonly userServiceProxy: ClientProxy,

apps/utility/src/utilities.module.ts

@@ -7,7 +7,7 @@ import { PrismaService } from '@credebl/prisma-service';
 import { UtilitiesController } from './utilities.controller';
 import { UtilitiesService } from './utilities.service';
 import { UtilitiesRepository } from './utilities.repository';
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { CommonConstants } from '@credebl/common/common.constant';
 import { GlobalConfigModule } from '@credebl/config/global-config.module';
 import { ConfigModule as PlatformConfig } from '@credebl/config/config.module';
@@ -35,6 +35,6 @@ import { ContextInterceptorModule } from '@credebl/context/contextInterceptorMod
     CacheModule.register()
   ],
   controllers: [UtilitiesController],
-  providers: [UtilitiesService, Logger, PrismaService, UtilitiesRepository, AwsService]
+  providers: [UtilitiesService, Logger, PrismaService, UtilitiesRepository, StorageService]
 })
 export class UtilitiesModule {}

apps/utility/src/utilities.service.ts

@@ -1,6 +1,6 @@
 import { Injectable, NotFoundException } from '@nestjs/common';
 
-import { AwsService } from '@credebl/aws';
+import { StorageService } from '@credebl/storage';
 import { BaseService } from 'libs/service/base.service';
 import { EmailDto } from '@credebl/common/dtos/email.dto';
 import { EmailService } from '@credebl/common/email.service';
@@ -17,7 +17,7 @@ export class UtilitiesService extends BaseService {
 
   constructor(
     private readonly utilitiesRepository: UtilitiesRepository,
-    private readonly awsService: AwsService,
+    private readonly awsService: StorageService,
     private readonly emailService: EmailService
   ) {
     super('UtilitiesService');
@@ -71,6 +71,10 @@ export class UtilitiesService extends BaseService {
         uuid,
         payload.storeObj
       );
+      const storageType = process.env.FILE_STORAGE_TYPE || 'aws';
+      if ('local' === storageType || 'rustfs' === storageType) {
+        return uploadResult.Location;
+      }
       const url: string = `${process.env.SHORTENED_URL_DOMAIN}/${uploadResult.Key}`;
       return url;
     } catch (error) {