Skip to content

Commit

Permalink
Change incorrect money types to integer - etl and seed update (#139519)
Browse files Browse the repository at this point in the history
Co-authored-by: Garrett Rabian <[email protected]>
  • Loading branch information
gmrabian and gmrabian authored Nov 21, 2023
1 parent 8dd8c5a commit 930e2ff
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 18 deletions.
12 changes: 6 additions & 6 deletions services/database/data/seed-local/seed-section.json
Original file line number Diff line number Diff line change
Expand Up @@ -11665,23 +11665,23 @@
"comment": "This is the first real question so far in this part…",
"id": "2022-05-a-03-01-a",
"label": "2022",
"type": "money",
"type": "integer",
"answer": {
"entry": "5"
}
},
{
"id": "2022-05-a-03-01-b",
"label": "2023",
"type": "money",
"type": "integer",
"answer": {
"entry": "5"
}
},
{
"id": "2022-05-a-03-01-c",
"label": "2024",
"type": "money",
"type": "integer",
"answer": {
"entry": "5"
}
Expand Down Expand Up @@ -11825,23 +11825,23 @@
"comment": "This is the first real question so far in this part…",
"id": "2022-05-a-04-01-a",
"label": "2022",
"type": "money",
"type": "integer",
"answer": {
"entry": "5"
}
},
{
"id": "2022-05-a-04-01-b",
"label": "2023",
"type": "money",
"type": "integer",
"answer": {
"entry": "5"
}
},
{
"id": "2022-05-a-04-01-c",
"label": "2024",
"type": "money",
"type": "integer",
"answer": {
"entry": "5"
}
Expand Down
12 changes: 6 additions & 6 deletions services/database/data/seed/seed-section-base-2022.json
Original file line number Diff line number Diff line change
Expand Up @@ -11513,7 +11513,7 @@
"questions": [
{
"id": "2022-05-a-03-01-a",
"type": "money",
"type": "integer",
"label": "2022",
"answer": {
"entry": null
Expand All @@ -11522,15 +11522,15 @@
},
{
"id": "2022-05-a-03-01-b",
"type": "money",
"type": "integer",
"label": "2023",
"answer": {
"entry": null
}
},
{
"id": "2022-05-a-03-01-c",
"type": "money",
"type": "integer",
"label": "2024",
"answer": {
"entry": null
Expand Down Expand Up @@ -11673,7 +11673,7 @@
"questions": [
{
"id": "2022-05-a-04-01-a",
"type": "money",
"type": "integer",
"label": "2022",
"answer": {
"entry": null
Expand All @@ -11682,15 +11682,15 @@
},
{
"id": "2022-05-a-04-01-b",
"type": "money",
"type": "integer",
"label": "2023",
"answer": {
"entry": null
}
},
{
"id": "2022-05-a-04-01-c",
"type": "money",
"type": "integer",
"label": "2024",
"answer": {
"entry": null
Expand Down
12 changes: 6 additions & 6 deletions services/database/data/seed/seed-section-base-2023.json
Original file line number Diff line number Diff line change
Expand Up @@ -11454,7 +11454,7 @@
"questions": [
{
"id": "2023-05-a-03-01-a",
"type": "money",
"type": "integer",
"label": "2023",
"answer": {
"entry": null
Expand All @@ -11463,15 +11463,15 @@
},
{
"id": "2023-05-a-03-01-b",
"type": "money",
"type": "integer",
"label": "2024",
"answer": {
"entry": null
}
},
{
"id": "2023-05-a-03-01-c",
"type": "money",
"type": "integer",
"label": "2025",
"answer": {
"entry": null
Expand Down Expand Up @@ -11614,7 +11614,7 @@
"questions": [
{
"id": "2023-05-a-04-01-a",
"type": "money",
"type": "integer",
"label": "2023",
"answer": {
"entry": null
Expand All @@ -11623,15 +11623,15 @@
},
{
"id": "2023-05-a-04-01-b",
"type": "money",
"type": "integer",
"label": "2024",
"answer": {
"entry": null
}
},
{
"id": "2023-05-a-04-01-c",
"type": "money",
"type": "integer",
"label": "2025",
"answer": {
"entry": null
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
let dynamoClient;
let dynamoPrefix;
// eslint-disable-next-line no-unused-vars
async function handler(event, context, callback) {
// eslint-disable-next-line no-console
console.log("Start of data fix");

const aws = require("aws-sdk");

const dynamoConfig = {};
const endpoint = process.env.DYNAMODB_URL;
if (endpoint) {
dynamoConfig.endpoint = endpoint;
dynamoConfig.accessKeyId = "LOCAL_FAKE_KEY"; // pragma: allowlist secret
dynamoConfig.secretAccessKey = "LOCAL_FAKE_SECRET"; // pragma: allowlist secret
dynamoPrefix = "local";
} else {
dynamoConfig["region"] = "us-east-1";
dynamoPrefix = process.env.dynamoPrefix;
}

dynamoClient = new aws.DynamoDB.DocumentClient(dynamoConfig);

const results = await scan(dynamoClient);
const transformed = await transform(results);
const keys = ["pk", "sectionId"];
await updateItems(`${dynamoPrefix}-section`, transformed, keys);

return results;
// console.log("Completed data fix");
}

async function transform(items) {
const transformed = items.map((item) => {
const corrected = { ...item };

// part 3
corrected.contents.section.subsections[0].parts[2].questions[0].questions[0].questions[0].type =
"integer";
corrected.contents.section.subsections[0].parts[2].questions[0].questions[0].questions[1].type =
"integer";
corrected.contents.section.subsections[0].parts[2].questions[0].questions[0].questions[2].type =
"integer";
// part 4
corrected.contents.section.subsections[0].parts[3].questions[0].questions[0].questions[0].type =
"integer";
corrected.contents.section.subsections[0].parts[3].questions[0].questions[0].questions[1].type =
"integer";
corrected.contents.section.subsections[0].parts[3].questions[0].questions[0].questions[2].type =
"integer";

return corrected;
});

return transformed;
}

async function scan(dynamoClient) {
let startingKey;
let existingItems = [];
let results;

const queryParams = {
TableName: `${dynamoPrefix}-section`,
ExpressionAttributeNames: {
"#year": "year",
"#sectionId": "sectionId",
},
ExpressionAttributeValues: {
":year": 2023,
":sectionId": 5,
},
FilterExpression: "#year = :year AND #sectionId = :sectionId",
};

const queryTable = async (startingKey) => {
queryParams.ExclusiveStartKey = startingKey;
let results = await dynamoClient.scan(queryParams).promise();
if (results.LastEvaluatedKey) {
startingKey = results.LastEvaluatedKey;
return [startingKey, results];
} else {
return [null, results];
}
};

// Looping to perform complete scan of tables due to 1 mb limit per iteration
do {
[startingKey, results] = await queryTable(startingKey);
const items = results?.Items;
existingItems.push(...items);
} while (startingKey);

return existingItems;
}

// eslint-disable-next-line no-unused-vars
const updateItems = async (tableName, items, keys) => {
try {
for (const item of items) {
let key = {};
for (const k of keys) {
key[k] = item[k];
delete item[k];
}

const params = {
TableName: tableName,
Key: key,
...convertToDynamoExpression(item),
};
await dynamoClient.update(params).promise();
}
} catch (e) {
// eslint-disable-next-line no-console
console.log(` -- ERROR UPLOADING ${tableName}\n`, e);
}
};

const convertToDynamoExpression = (listOfVars) => {
let expressionAttributeNames = {};
let expressionAttributeValues = {};
let updateExpression = "";
Object.keys(listOfVars).forEach((key, index) => {
expressionAttributeNames[`#${key}`] = key;
expressionAttributeValues[`:${key}`] = listOfVars[key];

updateExpression =
index === 0
? `set #${key}=:${key}`
: `${updateExpression}, #${key}=:${key}`;
});
return {
UpdateExpression: updateExpression,
ExpressionAttributeNames: expressionAttributeNames,
ExpressionAttributeValues: expressionAttributeValues,
};
};

exports.handler = handler;
5 changes: 5 additions & 0 deletions services/database/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,11 @@ functions:
dynamoPrefix: ${self:custom.stage}
seedTestData: ${self:custom.seedTestData}
timeout: 120
convertMoneyTypeToInteger:
handler: handlers/dataCorrections/convertMoneyTypeToInteger.handler
environment:
dynamoPrefix: ${self:custom.stage}
timeout: 120

resources:
Resources:
Expand Down

0 comments on commit 930e2ff

Please sign in to comment.