Skip to content

Commit

Permalink
API gateway timeout issue changes done in DM job.
Browse files Browse the repository at this point in the history
  • Loading branch information
@madhumathivenkatesan-bc
madhumathivenkatesan-bc committed Oct 12, 2023
1 parent 4001c71 commit 6168ae6
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 47 deletions.
1 change: 1 addition & 0 deletions api/CcsSso.Core.DataMigrationJobScheduler/Model/DataMigrationFileListResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public class DataMigrationFileListInfo : DataMigrationListInfo
public class DataMigrationFileListResponse
{
public List<DataMigrationFileListInfo> DataMigrationList { get; set; }
public int filesCount { get; set; }
}

public class DataMigrationStatusRequest
Expand Down
106 changes: 60 additions & 46 deletions api/CcsSso.Core.DataMigrationJobScheduler/Services/FileUploadService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,65 +38,79 @@ public async Task PerformFileUploadJobAsync()
{
try
{
int filesCount = 0;
var files = await _wrapperOrganisationService.GetDataMigrationFilesList();

if (files.DataMigrationList.Any())
filesCount = files.filesCount;
_logger.LogInformation($"****** Total no of files:{filesCount} *******");
if (filesCount>0)
{
_logger.LogInformation($"****** No of files found: {files.DataMigrationList.Count()}");

foreach (var file in files.DataMigrationList)
do
{
_logger.LogInformation($"****** Reading File from S3: {file.FileKey} ***********");
var fileContentString = await _awsS3Service.ReadObjectDataStringAsync(file.FileKey, _s3ConfigurationInfo.DataMigrationBucketName);
_logger.LogInformation($"****** Downloaded File from S3 : {file.FileKey} ***********");

var client = _httpClientFactory.CreateClient("DataMigrationApi");
using (var formData = new MultipartFormDataContent())
if (files.DataMigrationList.Any())
{
_logger.LogInformation($"****** Converting File string Data to Memory Stream : {file.FileKey} ***********");
using (var s3DataStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(fileContentString)))
{
// Create a StreamContent from the S3 data stream
var streamContent = new StreamContent(s3DataStream);
_logger.LogInformation($"****** No of files found: {files.DataMigrationList.Count()}");

// Add the StreamContent to the form data
formData.Add(streamContent, "file", "data.csv"); // You can specify the file name as needed
foreach (var file in files.DataMigrationList)
{
_logger.LogInformation($"****** Reading File from S3: {file.FileKey} ***********");
var fileContentString = await _awsS3Service.ReadObjectDataStringAsync(file.FileKey, _s3ConfigurationInfo.DataMigrationBucketName);
_logger.LogInformation($"****** Downloaded File from S3 : {file.FileKey} ***********");

_logger.LogInformation($"****** Uploading File to Data Migration API : {file.FileKey} ***********");
// Make a POST request to the CII API
var response = await client.PostAsync($"data-migration/migrate/format/csv", formData);
if (response.IsSuccessStatusCode)
{
_logger.LogInformation($"****** File Uploaded Successfully to Data Migration API : {file.FileKey} ***********");
var fileKey = GetUploadFileKey(file.FileKey, true);
_logger.LogInformation($"****** Moving file to Success Folder : {fileKey} ***********");
await _awsS3Service.WritingAFileDataAsync(fileKey, "text/csv", _s3ConfigurationInfo.DataMigrationBucketName, fileContentString);
_logger.LogInformation($"****** Updating File Status as Completed : {file.Id} ***********");
await _wrapperOrganisationService.UpdateDataMigrationFileStatus(new DataMigrationStatusRequest { Id = file.Id, DataMigrationStatus = DataMigrationStatus.Completed });
}
else if (response.StatusCode == HttpStatusCode.ServiceUnavailable)
var client = _httpClientFactory.CreateClient("DataMigrationApi");
using (var formData = new MultipartFormDataContent())
{
throw new CcsSsoException("CII_SERVICE_IS_TEMPORARILY_UNAVAILABLE");
}
else
{
_logger.LogInformation($"****** File Upload to Data Migration API failed : {file.FileKey} ***********");
var fileKey = GetUploadFileKey(file.FileKey, false);
_logger.LogInformation($"****** Moving file to Failed Folder : {fileKey} ***********");
await _awsS3Service.WritingAFileDataAsync(fileKey, "text/csv", _s3ConfigurationInfo.DataMigrationBucketName, fileContentString);
_logger.LogInformation($"****** Updating File Status as Failed : {file.Id} ***********");
await _wrapperOrganisationService.UpdateDataMigrationFileStatus(new DataMigrationStatusRequest { Id = file.Id, DataMigrationStatus = DataMigrationStatus.Failed });
_logger.LogInformation($"****** Converting File string Data to Memory Stream : {file.FileKey} ***********");
using (var s3DataStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(fileContentString)))
{
// Create a StreamContent from the S3 data stream
var streamContent = new StreamContent(s3DataStream);

// Add the StreamContent to the form data
formData.Add(streamContent, "file", "data.csv"); // You can specify the file name as needed

_logger.LogInformation($"****** Uploading File to Data Migration API : {file.FileKey} ***********");
// Make a POST request to the CII API
var response = await client.PostAsync($"data-migration/migrate/format/csv", formData);
if (response.IsSuccessStatusCode)
{
_logger.LogInformation($"****** File Uploaded Successfully to Data Migration API : {file.FileKey} ***********");
var fileKey = GetUploadFileKey(file.FileKey, true);
_logger.LogInformation($"****** Moving file to Success Folder : {fileKey} ***********");
await _awsS3Service.WritingAFileDataAsync(fileKey, "text/csv", _s3ConfigurationInfo.DataMigrationBucketName, fileContentString);
_logger.LogInformation($"****** Updating File Status as Completed : {file.Id} ***********");
await _wrapperOrganisationService.UpdateDataMigrationFileStatus(new DataMigrationStatusRequest { Id = file.Id, DataMigrationStatus = DataMigrationStatus.Completed });
}
else if (response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
throw new CcsSsoException("CII_SERVICE_IS_TEMPORARILY_UNAVAILABLE");
}
else
{
_logger.LogInformation($"****** File Upload to Data Migration API failed : {file.FileKey} ***********");
var fileKey = GetUploadFileKey(file.FileKey, false);
_logger.LogInformation($"****** Moving file to Failed Folder : {fileKey} ***********");
await _awsS3Service.WritingAFileDataAsync(fileKey, "text/csv", _s3ConfigurationInfo.DataMigrationBucketName, fileContentString);
_logger.LogInformation($"****** Updating File Status as Failed : {file.Id} ***********");
await _wrapperOrganisationService.UpdateDataMigrationFileStatus(new DataMigrationStatusRequest { Id = file.Id, DataMigrationStatus = DataMigrationStatus.Failed });
}
}
}
}
files= await _wrapperOrganisationService.GetDataMigrationFilesList();
filesCount = files.filesCount;
_logger.LogInformation($"****** Total no of files: {files.DataMigrationList.Count()} *******");
}
else
{
_logger.LogInformation($"****** No files found ******");
}
}
while ( filesCount>0 );


}
else
{
_logger.LogInformation($"****** No files found ******");
}
}
catch(Exception ex)
catch (Exception ex)
{
_logger.LogError($"Error While Running a Job. Error-{ex.Message}");
}
Expand Down
2 changes: 1 addition & 1 deletion api/CcsSso.Core.DataMigrationJobScheduler/Wrapper/WrapperOrganisationService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public WrapperOrganisationService(IWrapperApiService wrapperApiService)

public async Task<DataMigrationFileListResponse> GetDataMigrationFilesList()
{
return await _wrapperApiService.GetAsync<DataMigrationFileListResponse>(WrapperApi.Organisation, $"migrations/files/validated", "ERROR_RETRIEVING_DM_FILES",false);
return await _wrapperApiService.GetAsync<DataMigrationFileListResponse>(WrapperApi.Organisation, $"migrations/files/validated?PageSize=10&CurrentPage=1", "ERROR_RETRIEVING_DM_FILES",false);
}

public async Task UpdateDataMigrationFileStatus(DataMigrationStatusRequest dataMigrationStatusRequest)
Expand Down

0 comments on commit 6168ae6

Please sign in to comment.