Skip to content

Commit

Permalink
Add tabs in model analysis report.
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 570941191
  • Loading branch information
@achoum @copybara-github
achoum authored and copybara-github committed Oct 5, 2023
1 parent 6cd50c8 commit 0206955
Show file tree
Hide file tree
Showing 6 changed files with 207 additions and 83 deletions.
6 changes: 5 additions & 1 deletion yggdrasil_decision_forests/utils/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,19 +739,22 @@ cc_library_ydf(
":distribution",
":feature_importance",
":filesystem",
":histogram",
":html",
":model_analysis_cc_proto",
":partial_dependence_plot",
":partial_dependence_plot_cc_proto",
":plot",
":uid",
"//yggdrasil_decision_forests/dataset:data_spec",
"//yggdrasil_decision_forests/dataset:example_cc_proto",
"//yggdrasil_decision_forests/dataset:vertical_dataset",
"//yggdrasil_decision_forests/dataset:vertical_dataset_io",
"//yggdrasil_decision_forests/model:abstract_model",
"//yggdrasil_decision_forests/model:model_engine_wrapper",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/time",
],
)

Expand Down Expand Up @@ -811,6 +814,7 @@ all_proto_library(
name = "model_analysis_proto",
srcs = ["model_analysis.proto"],
deps = [
"//yggdrasil_decision_forests/dataset:data_spec_proto",
"//yggdrasil_decision_forests/model:abstract_model_proto",
"//yggdrasil_decision_forests/utils:partial_dependence_plot_proto",
],
Expand Down
4 changes: 4 additions & 0 deletions yggdrasil_decision_forests/utils/html.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,10 @@ inline internal::Attr Class(absl::string_view value) {
return internal::Attr("class", value);
}

inline internal::Attr OnClick(absl::string_view value) {
return internal::Attr("onclick", value);
}

inline internal::Attr Id(absl::string_view value) {
return internal::Attr("id", value);
}
Expand Down
6 changes: 6 additions & 0 deletions yggdrasil_decision_forests/utils/html_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ TEST(Html, Style) {
EXPECT_EQ(style.content(), "background-color:hsl(180, 50%, 50%);a:b;");
}

TEST(Html, OnClick) {
auto page = A(OnClick("f('hello');"), "world");
// Note: Surprisingly to me, escaping strings this was in js code works fine.
EXPECT_EQ(page.content(), "<a onclick=\"f(&#39;hello&#39;);\">world</a>");
}

} // namespace
} // namespace html
} // namespace utils
Expand Down
245 changes: 164 additions & 81 deletions yggdrasil_decision_forests/utils/model_analysis.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,16 @@

#include "yggdrasil_decision_forests/utils/model_analysis.h"

#include <string>
#include <vector>

#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "yggdrasil_decision_forests/dataset/data_spec.h"
#include "yggdrasil_decision_forests/dataset/vertical_dataset_io.h"
#include "yggdrasil_decision_forests/model/abstract_model.h"
#include "yggdrasil_decision_forests/model/model_engine_wrapper.h"
Expand All @@ -26,6 +35,7 @@
#include "yggdrasil_decision_forests/utils/partial_dependence_plot.h"
#include "yggdrasil_decision_forests/utils/partial_dependence_plot.pb.h"
#include "yggdrasil_decision_forests/utils/plot.h"
#include "yggdrasil_decision_forests/utils/uid.h"

namespace yggdrasil_decision_forests {
namespace utils {
Expand Down Expand Up @@ -747,131 +757,204 @@ absl::Status CreateHtmlReport(const model::AbstractModel& model,

absl::StatusOr<std::string> CreateHtmlReport(
const model::AbstractModel& model, const dataset::VerticalDataset& dataset,
absl::string_view model_path, absl::string_view dataset_path,
const absl::string_view model_path, const absl::string_view dataset_path,
const proto::AnalysisResult& analysis, const proto::Options& options) {
const auto standalone = CreateStandaloneAnalysis(model, dataset, model_path,
dataset_path, analysis);
return CreateHtmlReport(standalone, options);
}

proto::StandaloneAnalysisResult CreateStandaloneAnalysis(
const model::AbstractModel& model, const dataset::VerticalDataset& dataset,
const absl::string_view model_path, const absl::string_view dataset_path,
const proto::AnalysisResult& analysis) {
proto::StandaloneAnalysisResult standalone;
*standalone.mutable_core_analysis() = analysis;
standalone.set_dataset_path(std::string(model_path));
standalone.set_model_path(std::string(dataset_path));
*standalone.mutable_data_spec() = model.data_spec();
standalone.set_label_col_idx(model.label_col_idx());
standalone.set_task(model.task());
standalone.set_model_description(model.DescriptionAndStatistics(false));
return standalone;
}

std::string Header() {
return R"(
<style>
.tab_block .header {
flex-direction: row;
display: flex;
}
.tab_block .header .tab {
cursor: pointer;
background-color: #F6F5F5;
text-decoration: none;
text-align: center;
padding: 4px 12px;
color: black;
}
.tab_block .header .tab.selected {
border-bottom: 2px solid #2F80ED;
}
.tab_block .header .tab:hover {
text-decoration: none;
background-color: #DCDCDC;
}
.tab_block .body .content {
display: none;
}
.tab_block .body .content.selected {
display: block;
}
</style>
<script>
function ydfAnalysisShowTab(block_id, item) {
const block = document.getElementById(block_id);
block.getElementsByClassName("tab selected")[0].classList.remove("selected");
block.getElementsByClassName("content selected")[0].classList.remove("selected");
document.getElementById(block_id + "_" + item).classList.add("selected");
document.getElementById(block_id + "_body_" + item).classList.add("selected");
}
</script>
)";
}

absl::StatusOr<std::string> CreateHtmlReport(
const proto::StandaloneAnalysisResult& analysis,
const proto::Options& options) {
namespace h = utils::html;
const auto block_id = utils::GenUniqueId();

h::Html html;
html.AppendRaw(Header());

// Report header.
if (options.report_header().enabled()) {
h::Html report_header;
report_header.Append(h::H1("Model and Dataset Analysis Report"));
report_header.Append(h::P(
absl::FormatTime(absl::RFC3339_sec, absl::Now(),
absl::LocalTimeZone()), h::Br(),
"Report generated with Yggdrasil Decision Forests"));
report_header.Append(h::H1("Model Analysis"));
report_header.Append(h::P(absl::FormatTime(absl::RFC3339_sec, absl::Now(),
absl::LocalTimeZone())));
report_header.Append(
h::P("Report generated by Yggdrasil Decision Forests"));
html.Append(report_header);
}

// Table of content.
if (options.table_of_content().enabled()) {
h::Html report_toc;
report_toc.Append(h::H2("Table of Content"));
report_toc.Append(
h::Ul(h::Li(h::A(h::HRef("#report_setup"), "Report Setup")),
h::Li(h::A(h::HRef("#data_spec"), "Dataset Specification")),
h::Li(h::A(h::HRef("#partial_dependence_plot"),
"Partial Dependence Plot")),
h::Li(h::A(h::HRef("#conditional_expectation_plot"),
"Conditional Expectation Plot")),
h::Li(h::A(h::HRef("#var_importance"),
"Permutation Variable Importance on Analyse Dataset")),
h::Li(h::A(h::HRef("#model_description"), "Model Description"))));
html.Append(report_toc);
}

// Report Setup.
h::Html tab_header;
h::Html tab_content;

// Adds a tab to the page.
bool first_tab = true;
const auto add_tab = [&tab_header, &tab_content, &block_id, &first_tab](
const absl::string_view key,
const absl::string_view title,
const h::Html& content) {
const absl::string_view maybe_selected = first_tab ? " selected" : "";
const auto onclick =
absl::Substitute("ydfAnalysisShowTab('$0', '$1')", block_id, key);

tab_header.Append(h::A(h::Id(absl::StrCat(block_id, "_", key)),
h::Class(absl::StrCat("tab", maybe_selected)),
h::OnClick(onclick), title));
tab_content.Append(h::Div(h::Id(absl::StrCat(block_id, "_body_", key)),
h::Class(absl::StrCat("content", maybe_selected)),
content));

first_tab = false;
};

// Setup
if (options.report_setup().enabled()) {
h::Html report_setup;
report_setup.Append(h::H2(h::Id("report_setup"), "Report Setup"));
report_setup.Append(h::P(h::B("Analyse dataset: "), dataset_path, h::Br(),
h::B("Model: "), model_path));
html.Append(report_setup);
h::Html content;
content.Append(h::P(h::B("Analyse dataset: "), analysis.dataset_path()));
content.Append(h::P(h::B("Model: "), analysis.model_path()));
add_tab("setup", "Setup", content);
}

// Dataset Specification.
if (options.dataspec().enabled()) {
h::Html report_data_spec;
report_data_spec.Append(h::H2(h::Id("data_spec"), "Dataset Specification"));
report_data_spec.Append(
h::Pre(dataset::PrintHumanReadable(model.data_spec(), false)));
html.Append(report_data_spec);
// Dataset Specification
if (options.report_setup().enabled() && analysis.has_data_spec()) {
h::Html content;
content.Append(
h::Pre(dataset::PrintHumanReadable(analysis.data_spec(), false)));
add_tab("dataset", "Dataset", content);
}

// Partial Dependence Plot
if (options.pdp().enabled()) {
html.Append(
h::H2(h::Id("partial_dependence_plot"), "Partial Dependence Plot"));
if (options.pdp().enabled() && analysis.core_analysis().has_pdp_set()) {
utils::plot::ExportOptions plot_options;
plot_options.show_interactive_menu = options.plot().show_interactive_menu();
ASSIGN_OR_RETURN(const auto plot,
internal::PlotPartialDependencePlotSet(
model.data_spec(), analysis.pdp_set(), model.task(),
model.label_col_idx(), options, &plot_options.width,
&plot_options.height));
ASSIGN_OR_RETURN(
const auto plot,
internal::PlotPartialDependencePlotSet(
analysis.data_spec(), analysis.core_analysis().pdp_set(),
analysis.task(), analysis.label_col_idx(), options,
&plot_options.width, &plot_options.height));
ASSIGN_OR_RETURN(const auto multiplot_html,
utils::plot::ExportToHtml(plot, plot_options));
html.AppendRaw(multiplot_html);

h::Html content;
content.AppendRaw(multiplot_html);
add_tab("pdp", "Partial Dependence Plot", content);
}

// Conditional Expectation Plot
if (options.cep().enabled()) {
html.Append(h::H2(h::Id("conditional_expectation_plot"),
"Conditional Expectation Plot"));
if (options.cep().enabled() && analysis.core_analysis().has_cep_set()) {
utils::plot::ExportOptions plot_options;
plot_options.show_interactive_menu = options.plot().show_interactive_menu();
ASSIGN_OR_RETURN(const auto plot,
internal::PlotConditionalExpectationPlotSet(
model.data_spec(), analysis.cep_set(), model.task(),
model.label_col_idx(), options, &plot_options.width,
&plot_options.height));
ASSIGN_OR_RETURN(
const auto plot,
internal::PlotConditionalExpectationPlotSet(
analysis.data_spec(), analysis.core_analysis().cep_set(),
analysis.task(), analysis.label_col_idx(), options,
&plot_options.width, &plot_options.height));
ASSIGN_OR_RETURN(const auto multiplot_html,
utils::plot::ExportToHtml(plot, plot_options));
html.AppendRaw(multiplot_html);

h::Html content;
content.AppendRaw(multiplot_html);
add_tab("cep", "Conditional Expectation Plot", content);
}

// Permutation Variable Importance.
if (options.permuted_variable_importance().enabled()) {
h::Html report_data_spec;
report_data_spec.Append(
h::H2(h::Id("var_importance"),
"Permutation Variable Importance on Analyse Dataset"));

report_data_spec.Append(h::P(
"The following permutation variable importance measures are computed "
"on the dataset given as input to the :analyse_model_and_dataset "
"tool. The reported values can differ from the variable importance "
"measures provided by the learner itself (model description "
"section)."));
// Permutation Variable Importance
if (options.permuted_variable_importance().enabled() &&
analysis.core_analysis().variable_importances_size() > 0) {
h::Html content;

std::string raw_variable_importance;
for (const auto& variable_importance : analysis.variable_importances()) {
for (const auto& variable_importance :
analysis.core_analysis().variable_importances()) {
absl::StrAppend(&raw_variable_importance, variable_importance.first);
absl::StrAppend(&raw_variable_importance, "\n\n");
std::vector<model::proto::VariableImportance> variable_importance_values =
{variable_importance.second.variable_importances().begin(),
variable_importance.second.variable_importances().end()};
model::AppendVariableImportanceDescription(variable_importance_values,
model.data_spec(), 4,
analysis.data_spec(), 4,
&raw_variable_importance);
absl::StrAppend(&raw_variable_importance, "\n\n");
}
report_data_spec.Append(h::Pre(raw_variable_importance));
html.Append(report_data_spec);
content.Append(h::Pre(raw_variable_importance));
add_tab("pva", "Permutation Variable Importance", content);
}

// Model Description.
if (options.model_description().enabled()) {
h::Html report_model_description;
report_model_description.Append(
h::H2(h::Id("model_description"), "Model Description"));
std::string description;
model.AppendDescriptionAndStatistics(false, &description);
report_model_description.Append(h::Pre(description));

html.Append(report_model_description);
// Model Description
if (options.model_description().enabled() &&
analysis.has_model_description()) {
h::Html content;
content.Append(h::Pre(analysis.model_description()));
add_tab("model", "Model Description", content);
}

html.Append(h::Div(h::Class("tab_block"), h::Id(block_id),
h::Div(h::Class("header"), tab_header),
h::Div(h::Class("body"), tab_content)));

return std::string(html.content());
}

Expand Down
15 changes: 15 additions & 0 deletions yggdrasil_decision_forests/utils/model_analysis.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,13 @@
#ifndef YGGDRASIL_DECISION_FORESTS_UTILS_MODEL_ANALYSIS_H_
#define YGGDRASIL_DECISION_FORESTS_UTILS_MODEL_ANALYSIS_H_

#include <string>

#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "yggdrasil_decision_forests/dataset/example.pb.h"
#include "yggdrasil_decision_forests/dataset/vertical_dataset.h"
#include "yggdrasil_decision_forests/model/abstract_model.h"
#include "yggdrasil_decision_forests/utils/model_analysis.pb.h"
#include "yggdrasil_decision_forests/utils/partial_dependence_plot.h"
Expand Down Expand Up @@ -50,6 +54,17 @@ absl::StatusOr<std::string> CreateHtmlReport(
absl::string_view model_path, absl::string_view dataset_path,
const proto::AnalysisResult& analysis, const proto::Options& options = {});

// Same as "CreateHtmlReport" from a standalone analysis.
absl::StatusOr<std::string> CreateHtmlReport(
const proto::StandaloneAnalysisResult& analysis,
const proto::Options& options = {});

// Assemble an analysis into a standalone analysis.
proto::StandaloneAnalysisResult CreateStandaloneAnalysis(
const model::AbstractModel& model, const dataset::VerticalDataset& dataset,
absl::string_view model_path, absl::string_view dataset_path,
const proto::AnalysisResult& analysis);

// Combines the model analysis and html creation i.e. Analyse +
// CreateHtmlReport.
absl::Status AnalyseAndCreateHtmlReport(const model::AbstractModel& model,
Expand Down
Loading

0 comments on commit 0206955

Please sign in to comment.