Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support EXPLAIN ... FORMAT <indent | tree | json | graphviz > ... #15166

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Support EXPLAIN ... FORMAT <indent | tree | json | graphviz > ... #15166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c965e8e
Support EXPLAIN FORMAT <format>
alamb Mar 11, 2025
9a93ec1
Update datafusion/sql/src/parser.rs
alamb Mar 12, 2025
398cf18
Improve documentation
alamb Mar 12, 2025
114bf4b
Merge branch 'alamb/explan_format' of github.com:alamb/datafusion int…
alamb Mar 12, 2025
a83ec4e
Remove to_stringified and simplify code
alamb Mar 13, 2025
f2cbf9d
mark deprecated rather than remove
alamb Mar 13, 2025
fe08421
Merge remote-tracking branch 'apache/main' into alamb/explan_format
alamb Mar 13, 2025
31f402d
Add a note about explain format configuration
alamb Mar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 5 additions & 23 deletions datafusion-examples/examples/planner_api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
// under the License.

use datafusion::error::Result;
use datafusion::logical_expr::{LogicalPlan, PlanType};
use datafusion::physical_plan::{displayable, DisplayFormatType};
use datafusion::logical_expr::LogicalPlan;
use datafusion::physical_plan::displayable;
use datafusion::physical_planner::DefaultPhysicalPlanner;
use datafusion::prelude::*;

Expand Down Expand Up @@ -77,13 +77,7 @@ async fn to_physical_plan_in_one_api_demo(

println!(
"Physical plan direct from logical plan:\n\n{}\n\n",
displayable(physical_plan.as_ref())
.to_stringified(
false,
PlanType::InitialPhysicalPlan,
DisplayFormatType::Default
)
.plan
displayable(physical_plan.as_ref()).indent(false)
);

Ok(())
Expand Down Expand Up @@ -123,13 +117,7 @@ async fn to_physical_plan_step_by_step_demo(
.await?;
println!(
"Final physical plan:\n\n{}\n\n",
displayable(physical_plan.as_ref())
.to_stringified(
false,
PlanType::InitialPhysicalPlan,
DisplayFormatType::Default
)
.plan
displayable(physical_plan.as_ref()).indent(false)
);

// Call the physical optimizer with an existing physical plan (in this
Expand All @@ -142,13 +130,7 @@ async fn to_physical_plan_step_by_step_demo(
planner.optimize_physical_plan(physical_plan, &ctx.state(), |_, _| {})?;
println!(
"Optimized physical plan:\n\n{}\n\n",
displayable(physical_plan.as_ref())
.to_stringified(
false,
PlanType::InitialPhysicalPlan,
DisplayFormatType::Default
)
.plan
displayable(physical_plan.as_ref()).indent(false)
);

Ok(())
Expand Down
2 changes: 2 additions & 0 deletions datafusion/core/src/execution/session_state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,7 @@ impl SessionState {
return Ok(LogicalPlan::Explain(Explain {
verbose: e.verbose,
plan: Arc::clone(&e.plan),
explain_format: e.explain_format.clone(),
stringified_plans,
schema: Arc::clone(&e.schema),
logical_optimization_succeeded: false,
Expand Down Expand Up @@ -612,6 +613,7 @@ impl SessionState {

Ok(LogicalPlan::Explain(Explain {
verbose: e.verbose,
explain_format: e.explain_format.clone(),
plan,
stringified_plans,
schema: Arc::clone(&e.schema),
Expand Down
138 changes: 85 additions & 53 deletions datafusion/core/src/physical_planner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

use std::borrow::Cow;
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::Arc;

use crate::datasource::file_format::file_type_to_format;
Expand Down Expand Up @@ -78,8 +77,8 @@ use datafusion_expr::expr::{
use datafusion_expr::expr_rewriter::unnormalize_cols;
use datafusion_expr::logical_plan::builder::wrap_projection_for_join_if_necessary;
use datafusion_expr::{
Analyze, DescribeTable, DmlStatement, Explain, Extension, FetchType, Filter,
JoinType, RecursiveQuery, SkipType, SortExpr, StringifiedPlan, WindowFrame,
Analyze, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension, FetchType,
Filter, JoinType, RecursiveQuery, SkipType, SortExpr, StringifiedPlan, WindowFrame,
WindowFrameBound, WriteOp,
};
use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
Expand All @@ -89,7 +88,6 @@ use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::execution_plan::InvariantLevel;
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::unnest::ListUnnest;
use datafusion_physical_plan::DisplayFormatType;

use crate::schema_equivalence::schema_satisfied_by;
use async_trait::async_trait;
Expand Down Expand Up @@ -1740,12 +1738,54 @@ impl DefaultPhysicalPlanner {
let mut stringified_plans = vec![];

let config = &session_state.config_options().explain;
let explain_format = DisplayFormatType::from_str(&config.format)?;
let explain_format = &e.explain_format;

match explain_format {
ExplainFormat::Indent => { /* fall through */ }
ExplainFormat::Tree => {
// Tree render does not try to explain errors,
let physical_plan = self
.create_initial_plan(e.plan.as_ref(), session_state)
.await?;

let optimized_plan = self.optimize_physical_plan(
physical_plan,
session_state,
|_plan, _optimizer| {},
)?;

stringified_plans.push(StringifiedPlan::new(
FinalPhysicalPlan,
displayable(optimized_plan.as_ref())
.tree_render()
.to_string(),
));
}
ExplainFormat::PostgresJSON => {
stringified_plans.push(StringifiedPlan::new(
FinalLogicalPlan,
e.plan.display_pg_json().to_string(),
));
}
ExplainFormat::Graphviz => {
stringified_plans.push(StringifiedPlan::new(
FinalLogicalPlan,
e.plan.display_graphviz().to_string(),
));
}
};

let skip_logical_plan =
config.physical_plan_only || explain_format == DisplayFormatType::TreeRender;
if !stringified_plans.is_empty() {
return Ok(Arc::new(ExplainExec::new(
Arc::clone(e.schema.inner()),
stringified_plans,
e.verbose,
)));
}

if !skip_logical_plan {
// The indent mode is quite sophisticated, and handles quite a few
// different cases / options for displaying the plan.
if !config.physical_plan_only {
stringified_plans.clone_from(&e.stringified_plans);
if e.logical_optimization_succeeded {
stringified_plans.push(e.plan.to_stringified(FinalLogicalPlan));
Expand All @@ -1759,41 +1799,35 @@ impl DefaultPhysicalPlanner {
{
Ok(input) => {
// Include statistics / schema if enabled
stringified_plans.push(
stringified_plans.push(StringifiedPlan::new(
InitialPhysicalPlan,
displayable(input.as_ref())
.set_show_statistics(config.show_statistics)
.set_show_schema(config.show_schema)
.to_stringified(
e.verbose,
InitialPhysicalPlan,
explain_format,
),
);
.indent(e.verbose)
.to_string(),
));

// Show statistics + schema in verbose output even if not
// explicitly requested
if e.verbose {
if !config.show_statistics {
stringified_plans.push(
stringified_plans.push(StringifiedPlan::new(
InitialPhysicalPlanWithStats,
displayable(input.as_ref())
.set_show_statistics(true)
.to_stringified(
e.verbose,
InitialPhysicalPlanWithStats,
explain_format,
),
);
.indent(e.verbose)
.to_string(),
));
}
if !config.show_schema {
stringified_plans.push(
stringified_plans.push(StringifiedPlan::new(
InitialPhysicalPlanWithSchema,
displayable(input.as_ref())
.set_show_schema(true)
.to_stringified(
e.verbose,
InitialPhysicalPlanWithSchema,
explain_format,
),
);
.indent(e.verbose)
.to_string(),
));
}
}

Expand All @@ -1803,52 +1837,50 @@ impl DefaultPhysicalPlanner {
|plan, optimizer| {
let optimizer_name = optimizer.name().to_string();
let plan_type = OptimizedPhysicalPlan { optimizer_name };
stringified_plans.push(
stringified_plans.push(StringifiedPlan::new(
plan_type,
displayable(plan)
.set_show_statistics(config.show_statistics)
.set_show_schema(config.show_schema)
.to_stringified(e.verbose, plan_type, explain_format),
);
.indent(e.verbose)
.to_string(),
));
},
);
match optimized_plan {
Ok(input) => {
// This plan will includes statistics if show_statistics is on
stringified_plans.push(
stringified_plans.push(StringifiedPlan::new(
FinalPhysicalPlan,
displayable(input.as_ref())
.set_show_statistics(config.show_statistics)
.set_show_schema(config.show_schema)
.to_stringified(
e.verbose,
FinalPhysicalPlan,
explain_format,
),
);
.indent(e.verbose)
.to_string(),
));

// Show statistics + schema in verbose output even if not
// explicitly requested
if e.verbose {
if !config.show_statistics {
stringified_plans.push(
stringified_plans.push(StringifiedPlan::new(
FinalPhysicalPlanWithStats,
displayable(input.as_ref())
.set_show_statistics(true)
.to_stringified(
e.verbose,
FinalPhysicalPlanWithStats,
explain_format,
),
);
.indent(e.verbose)
.to_string(),
));
}
if !config.show_schema {
stringified_plans.push(
stringified_plans.push(StringifiedPlan::new(
FinalPhysicalPlanWithSchema,
// This will include schema if show_schema is on
// and will be set to true if verbose is on
displayable(input.as_ref())
.set_show_schema(true)
.to_stringified(
e.verbose,
FinalPhysicalPlanWithSchema,
explain_format,
),
);
.indent(e.verbose)
.to_string(),
));
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion datafusion/expr/src/logical_plan/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ use crate::{
};

use super::dml::InsertOp;
use super::plan::ColumnUnnestList;
use super::plan::{ColumnUnnestList, ExplainFormat};
use arrow::compute::can_cast_types;
use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};
use datafusion_common::display::ToStringifiedPlan;
Expand Down Expand Up @@ -1211,6 +1211,7 @@ impl LogicalPlanBuilder {
Ok(Self::new(LogicalPlan::Explain(Explain {
verbose,
plan: self.plan,
explain_format: ExplainFormat::Indent,
stringified_plans,
schema,
logical_optimization_succeeded: false,
Expand Down
6 changes: 3 additions & 3 deletions datafusion/expr/src/logical_plan/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ pub use ddl::{
pub use dml::{DmlStatement, WriteOp};
pub use plan::{
projection_schema, Aggregate, Analyze, ColumnUnnestList, DescribeTable, Distinct,
DistinctOn, EmptyRelation, Explain, Extension, FetchType, Filter, Join,
JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, Projection,
RecursiveQuery, Repartition, SkipType, Sort, StringifiedPlan, Subquery,
DistinctOn, EmptyRelation, Explain, ExplainFormat, Extension, FetchType, Filter,
Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType,
Projection, RecursiveQuery, Repartition, SkipType, Sort, StringifiedPlan, Subquery,
SubqueryAlias, TableScan, ToStringifiedPlan, Union, Unnest, Values, Window,
};
pub use statement::{
Expand Down
Loading