Skip to content

Commit

Permalink
Merge pull request #26 from C-Accel-CRIPT/fix/bcdb-properties
Browse files Browse the repository at this point in the history 
hack(bcdb): comment 'method' on every single property
  • Loading branch information
@bearmit
bearmit authored Aug 14, 2023
2 parents 9086851 + ad00d30 commit e6becfb
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 53 deletions.
68 changes: 34 additions & 34 deletions scripts/typescript/src/bcdb/bcdb.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import * as XLSX from "xlsx";
import { Column } from "./types/column";
import { ICitation, ICollection, ICondition, IMaterial, IProject, IProperty, IReference } from "@cript";
import { CriptGraphOptimizer, CriptValidator, LogLevel, Logger, LoggerOptions } from "@utilities";
import { CriptGraphOptimizer, CriptValidator, LogLevel, Logger, LoggerOptions, OptimizedProject } from "@utilities";

export class BCDBLoader {
readonly logger: Logger;
Expand All @@ -34,7 +34,7 @@ export class BCDBLoader {
input_file_path: string;
sheets: ['blocks', 'diblock']; // added for the user to understand, but only expect a const array
limitSheetRows: number;
}): Promise<IProject> {
}): Promise<OptimizedProject> {

//-- Download DB schema
try {
Expand Down Expand Up @@ -103,30 +103,30 @@ export class BCDBLoader {

let index = 0;
const arr = rows.slice(2) /** Skip 2 header rows */;
for(let row in arr) {
for(let row of arr) {
index++; // 1-based index
index % 100 == 0 && process.stdout.write(`-- Converting rows ... ${index}/${arr.length - 1}\n`);

this.logger.prefix = `[row ${String(index).padEnd(4)}]`;
//-- reference
// to store DOI and ORCIDs
const doi = row[Column.DOI];
const author = row[Column.ORCID];
if(!doi) {
throw new Error(`DOI undefined`);
}
let reference: IReference | undefined = references.get(doi);
if (!reference) {
const author = row[Column.ORCID];

reference = {
title: 'Reference',
title: doi,
node: ["Reference"],
doi: doi,
author: author ? [author] : undefined,
type: "journal_article",
};
if (reference.doi) {
this.validator.validate_or_throw(reference);
references.set(reference.doi, reference);
} else {
this.logger.error(`Unable to store this reference, it has no doi (we use it as a key)`, JSON.stringify(reference));
}
this.validator.validate_or_throw(reference);
references.set(doi, reference);
}
//log(`Reference read: ${reference.doi}`);

Expand Down Expand Up @@ -178,7 +178,7 @@ export class BCDBLoader {
key: "mw_n",
type: 'value',
value: row[Column.Mn],
method: row[Column.Mn_method],
//method: row[Column.Mn_method], TODO: fix data to match with vocab
unit: "g/mol",
condition: [temperature],
citation,
Expand All @@ -188,7 +188,7 @@ export class BCDBLoader {
key: "mw_w",
type: 'value',
value: row[Column.Mw],
method: row[Column.Mw_method],
//method: row[Column.Mw_method], TODO: fix data to match with vocab
unit: "g/mol",
condition: [temperature],
citation,
Expand All @@ -198,7 +198,7 @@ export class BCDBLoader {
key: "mw_d",
type: 'value',
value: row[Column.D],
method: row[Column.D_method],
//method: row[Column.D_method], TODO: fix data to match with vocab
unit: "g/mol",
condition: [temperature],
citation,
Expand All @@ -208,7 +208,7 @@ export class BCDBLoader {
key: "invariant_degree_of_polymerization",
type: 'value',
value: row[Column.N],
method: row[Column.N_method],
//method: row[Column.N_method], TODO: fix data to match with vocab
condition: [temperature],
citation,
unit: null,
Expand All @@ -227,7 +227,7 @@ export class BCDBLoader {
type: 'value',
notes: "phase1",
value: row[Column.PHASE1],
method: phase_method,
//method: phase_method, TODO: fix data to match with vocab
citation,
unit: null,
});
Expand All @@ -237,7 +237,7 @@ export class BCDBLoader {
notes: "phase2",
type: 'value',
value: row[Column.PHASE2],
method: phase_method,
//method: phase_method, TODO: fix data to match with vocab
citation,
unit: null,
});
Expand All @@ -255,7 +255,7 @@ export class BCDBLoader {
key: "mw_w",
type: 'value',
value: row[Column.Mw1],
method: row[Column.Mw1_method],
//method: row[Column.Mw1_method], TODO: fix data to match with vocab
unit: "g/mol",
citation,
});
Expand All @@ -264,7 +264,7 @@ export class BCDBLoader {
key: "mw_d",
type: 'value',
value: row[Column.D1],
method: row[Column.D1_method],
//method: row[Column.D1_method], TODO: fix data to match with vocab
unit: "g/mol",
citation,
});
Expand All @@ -273,7 +273,7 @@ export class BCDBLoader {
key: "invariant_degree_of_polymerization",
type: 'value',
value: row[Column.N1],
method: row[Column.N1_method],
//method: row[Column.N1_method], TODO: fix data to match with vocab
unit: null,
citation,
});
Expand All @@ -282,7 +282,7 @@ export class BCDBLoader {
key: "conc_vol_fraction",
type: 'value',
value: row[Column.f1],
method: row[Column.f1_method],
//method: row[Column.f1_method], TODO: fix data to match with vocab
unit: null,
citation,
});
Expand All @@ -291,7 +291,7 @@ export class BCDBLoader {
key: "conc_vol_fraction",
type: 'value',
value: row[Column.ftot1],
method: row[Column.ftot1_method],
// method: row[Column.ftot1_method], TODO: fix data to match with vocab
unit: null,
citation,
});
Expand All @@ -300,7 +300,7 @@ export class BCDBLoader {
key: "conc_mass_fraction",
type: 'value',
value: row[Column.w1],
method: row[Column.w1_method],
//method: row[Column.w1_method], TODO: fix data to match with vocab
unit: null,
citation,
});
Expand All @@ -309,7 +309,7 @@ export class BCDBLoader {
key: "density",
type: 'value',
value: row[Column.rho1],
method: row[Column.rho1_method],
//method: row[Column.rho1_method], TODO: fix data to match with vocab
unit: "g/mL",
citation,
});
Expand All @@ -318,7 +318,7 @@ export class BCDBLoader {
key: "microstructure_phase",
type: 'value',
value: `${row[Column.PHASE1]},${row[Column.PHASE2]}`,
method: row[Column.rho1_method],
// method: row[Column.rho1_method], TODO: fix data to match with vocab
unit: "g/mL",
citation,
})
Expand Down Expand Up @@ -356,7 +356,7 @@ export class BCDBLoader {
key: "invariant_degree_of_polymerization",
type: 'value',
value: row[Column.N2],
method: row[Column.N2_method],
// method: row[Column.N2_method], TODO: fix data to match with vocab
unit: null,
citation,
});
Expand All @@ -365,7 +365,7 @@ export class BCDBLoader {
key: "conc_vol_fraction",
type: 'value',
value: row[Column.f2],
method: row[Column.f2_method],
//method: row[Column.f2_method], TODO: fix data to match with vocab
unit: null,
citation,
});
Expand All @@ -374,7 +374,7 @@ export class BCDBLoader {
key: "conc_vol_fraction",
type: 'value',
value: row[Column.ftot2],
method: row[Column.ftot2_method],
//method: row[Column.ftot2_method], TODO: fix data to match with vocab
unit: null,
citation,
});
Expand All @@ -383,7 +383,7 @@ export class BCDBLoader {
key: "conc_mass_fraction",
type: 'value',
value: row[Column.w2],
method: row[Column.w2_method],
// method: row[Column.w2_method], TODO: fix data to match with vocab
unit: null,
citation,
})
Expand All @@ -392,7 +392,7 @@ export class BCDBLoader {
key: "density",
type: 'value',
value: row[Column.rho2],
method: row[Column.rho2_method],
//method: row[Column.rho2_method], TODO: fix data to match with vocab
unit: "g/mL",
citation,
})
Expand Down Expand Up @@ -426,17 +426,17 @@ export class BCDBLoader {

// Optimise the project object (uses uids, create Edges, etc..)
const optimizer = new CriptGraphOptimizer();
const optimized_project: IProject = optimizer.get_optimized(project);
const optimized_project: OptimizedProject = optimizer.get_optimized(project);

// Validate against DB schema
const is_valid = await this.validator.validate('ProjectPost', optimized_project);
const is_valid = await this.validator.validate('ProjectPost', optimized_project.project);

if(!is_valid) {
this.logger.error(this.validator.errorsAsString(100));
this.logger.error(`Project '${optimized_project.name}' is NOT valid, see errors in logs above!`)
this.logger.error(`Project '${optimized_project.project.name}' is NOT valid, see errors in logs above!`)
throw new Error(`Project is NOT valid`);
} else {
this.logger.info(`Project '${optimized_project.name}' is valid.`)
this.logger.info(`Project '${optimized_project.project.name}' is valid.`)
}

return optimized_project;
Expand Down
2 changes: 1 addition & 1 deletion scripts/typescript/src/bcdb/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { BCDBLoader } from "./bcdb";

const loader = new BCDBLoader({ logger: {
outstream: logStream, // We use a stream to reduce considerably CPU usage compared to console.log|warn|error|info() which flushes after each call.
verbosity: LogLevel.DEBUG, // Consider reducing the verbosity when needed, it will reduce log output
verbosity: LogLevel.INFO, // Consider reducing the verbosity when needed, it will reduce log output
timestamp: true, // Not required, but usefull to figure out when a problem occured.
}});

Expand Down
6 changes: 3 additions & 3 deletions scripts/typescript/src/pppdb/pppdb.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { resolve } from "path";
import { ICitation, ICollection, ICondition, IMaterial, IProject, IProperty, IReference } from "@cript";
import { Chi, Method, Polymer, PubChemCASResponse, PubChemResponse as PubChemPropertyResponse, Solvent } from "./types";
import { molfile_to_bigsmiles } from "@cript-web/bigsmiles-toolkit";
import { CriptValidator, Logger, LogLevel, LoggerOptions, CriptGraphOptimizer } from "@utilities";
import { CriptValidator, Logger, LogLevel, LoggerOptions, CriptGraphOptimizer, OptimizedProject } from "@utilities";
import { Other } from "./types/sheets/others";
import { fetch } from "cross-fetch";
export class PPPDBLoader {
Expand Down Expand Up @@ -102,7 +102,7 @@ export class PPPDBLoader {
molfile_dir: string,
};
row_limit: number;
}): Promise<IProject> {
}): Promise<OptimizedProject> {
this.logger.prefix = null;
this.logger.info(`PPPDB.load() ...`);
this.logger.debug(`Reset state`);
Expand Down Expand Up @@ -531,7 +531,7 @@ export class PPPDBLoader {

// Validate the project using DB schema
this.logger.info(`Project validation ...`);
const project_is_valid = this.validator.validate('ProjectPost', optimized_project);
const project_is_valid = this.validator.validate('ProjectPost', optimized_project.project);
if( project_is_valid ) {
this.logger.info(`Project validation: SUCCEEDED!`);
} else {
Expand Down
Loading

0 comments on commit e6becfb

Please sign in to comment.