fix!: columns table in information_schema misses some columns

src/catalog/Cargo.toml

@@ -40,6 +40,7 @@ prometheus.workspace = true
 serde_json.workspace = true
 session.workspace = true
 snafu.workspace = true
+sql.workspace = true
 store-api.workspace = true
 table.workspace = true
 tokio.workspace = true

src/catalog/src/information_schema/columns.rs

@@ -30,9 +30,10 @@ use datatypes::prelude::{ConcreteDataType, DataType};
 use datatypes::scalars::ScalarVectorBuilder;
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::value::Value;
-use datatypes::vectors::{StringVectorBuilder, VectorRef};
+use datatypes::vectors::{Int64VectorBuilder, StringVectorBuilder, VectorRef};
 use futures::TryStreamExt;
 use snafu::{OptionExt, ResultExt};
+use sql::statements;
 use store_api::storage::{ScanRequest, TableId};
 
 use super::{InformationTable, COLUMNS};
@@ -52,6 +53,20 @@ pub const TABLE_CATALOG: &str = "table_catalog";
 pub const TABLE_SCHEMA: &str = "table_schema";
 pub const TABLE_NAME: &str = "table_name";
 pub const COLUMN_NAME: &str = "column_name";
+const ORDINAL_POSITION: &str = "ordinal_position";
+const CHARACTER_MAXIMUM_LENGTH: &str = "character_maximum_length";
+const CHARACTER_OCTET_LENGTH: &str = "character_octet_length";
+const NUMERIC_PRECISION: &str = "numeric_precision";
+const NUMERIC_SCALE: &str = "numeric_scale";
+const DATETIME_PRECISION: &str = "datetime_precision";
+const CHARACTER_SET_NAME: &str = "character_set_name";
+pub const COLLATION_NAME: &str = "collation_name";
+pub const COLUMN_KEY: &str = "column_key";
+pub const EXTRA: &str = "extra";
+pub const PRIVILEGES: &str = "privileges";
+const GENERATION_EXPRESSION: &str = "generation_expression";
+// Extension field to keep greptime data type name
+pub const GREPTIME_DATA_TYPE: &str = "greptime_data_type";
 pub const DATA_TYPE: &str = "data_type";
 pub const SEMANTIC_TYPE: &str = "semantic_type";
 pub const COLUMN_DEFAULT: &str = "column_default";
@@ -60,6 +75,15 @@ const COLUMN_TYPE: &str = "column_type";
 pub const COLUMN_COMMENT: &str = "column_comment";
 const INIT_CAPACITY: usize = 42;
 
+// The maximum length of string type
+const MAX_STRING_LENGTH: i64 = 2147483647;
+const UTF8_CHARSET_NAME: &str = "utf8";
+const UTF8_COLLATE_NAME: &str = "utf8_bin";
+const PRI_COLUMN_KEY: &str = "PRI";
+const TIME_INDEX_COLUMN_KEY: &str = "TIME INDEX";
+const DEFAULT_PRIVILEGES: &str = "select,insert";
+const EMPTY_STR: &str = "";
+
 impl InformationSchemaColumns {
     pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
         Self {
@@ -75,6 +99,39 @@ impl InformationSchemaColumns {
             ColumnSchema::new(TABLE_SCHEMA, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(TABLE_NAME, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(COLUMN_NAME, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(ORDINAL_POSITION, ConcreteDataType::int64_datatype(), false),
+            ColumnSchema::new(
+                CHARACTER_MAXIMUM_LENGTH,
+                ConcreteDataType::int64_datatype(),
+                true,
+            ),
+            ColumnSchema::new(
+                CHARACTER_OCTET_LENGTH,
+                ConcreteDataType::int64_datatype(),
+                true,
+            ),
+            ColumnSchema::new(NUMERIC_PRECISION, ConcreteDataType::int64_datatype(), true),
+            ColumnSchema::new(NUMERIC_SCALE, ConcreteDataType::int64_datatype(), true),
+            ColumnSchema::new(DATETIME_PRECISION, ConcreteDataType::int64_datatype(), true),
+            ColumnSchema::new(
+                CHARACTER_SET_NAME,
+                ConcreteDataType::string_datatype(),
+                true,
+            ),
+            ColumnSchema::new(COLLATION_NAME, ConcreteDataType::string_datatype(), true),
+            ColumnSchema::new(COLUMN_KEY, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(EXTRA, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(PRIVILEGES, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(
+                GENERATION_EXPRESSION,
+                ConcreteDataType::string_datatype(),
+                false,
+            ),
+            ColumnSchema::new(
+                GREPTIME_DATA_TYPE,
+                ConcreteDataType::string_datatype(),
+                false,
+            ),
             ColumnSchema::new(DATA_TYPE, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(SEMANTIC_TYPE, ConcreteDataType::string_datatype(), false),
             ColumnSchema::new(COLUMN_DEFAULT, ConcreteDataType::string_datatype(), true),
@@ -136,9 +193,21 @@ struct InformationSchemaColumnsBuilder {
     schema_names: StringVectorBuilder,
     table_names: StringVectorBuilder,
     column_names: StringVectorBuilder,
+    ordinal_positions: Int64VectorBuilder,
+    character_maximum_lengths: Int64VectorBuilder,
+    character_octet_lengths: Int64VectorBuilder,
+    numeric_precisions: Int64VectorBuilder,
+    numeric_scales: Int64VectorBuilder,
+    datetime_precisions: Int64VectorBuilder,
+    character_set_names: StringVectorBuilder,
+    collation_names: StringVectorBuilder,
+    column_keys: StringVectorBuilder,
+    extras: StringVectorBuilder,
+    privileges: StringVectorBuilder,
+    generation_expressions: StringVectorBuilder,
+    greptime_data_types: StringVectorBuilder,
     data_types: StringVectorBuilder,
     semantic_types: StringVectorBuilder,
-
     column_defaults: StringVectorBuilder,
     is_nullables: StringVectorBuilder,
     column_types: StringVectorBuilder,
@@ -159,6 +228,19 @@ impl InformationSchemaColumnsBuilder {
             schema_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             table_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             column_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            ordinal_positions: Int64VectorBuilder::with_capacity(INIT_CAPACITY),
+            character_maximum_lengths: Int64VectorBuilder::with_capacity(INIT_CAPACITY),
+            character_octet_lengths: Int64VectorBuilder::with_capacity(INIT_CAPACITY),
+            numeric_precisions: Int64VectorBuilder::with_capacity(INIT_CAPACITY),
+            numeric_scales: Int64VectorBuilder::with_capacity(INIT_CAPACITY),
+            datetime_precisions: Int64VectorBuilder::with_capacity(INIT_CAPACITY),
+            character_set_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            collation_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            column_keys: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            extras: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            privileges: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            generation_expressions: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            greptime_data_types: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             data_types: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             semantic_types: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             column_defaults: StringVectorBuilder::with_capacity(INIT_CAPACITY),
@@ -194,6 +276,7 @@ impl InformationSchemaColumnsBuilder {
                     };
 
                     self.add_column(
+                        idx,
                         &predicates,
                         &catalog_name,
                         &schema_name,
@@ -208,16 +291,27 @@ impl InformationSchemaColumnsBuilder {
         self.finish()
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn add_column(
         &mut self,
+        index: usize,
         predicates: &Predicates,
         catalog_name: &str,
         schema_name: &str,
         table_name: &str,
         semantic_type: &str,
         column_schema: &ColumnSchema,
     ) {
-        let data_type = &column_schema.data_type.name();
+        // Use sql data type name
+        let data_type = statements::concrete_data_type_to_sql_data_type(&column_schema.data_type)
+            .map(|dt| dt.to_string().to_lowercase())
+            .unwrap_or_else(|_| column_schema.data_type.name());
+
+        let column_key = match semantic_type {
+            SEMANTIC_TYPE_PRIMARY_KEY => PRI_COLUMN_KEY,
+            SEMANTIC_TYPE_TIME_INDEX => TIME_INDEX_COLUMN_KEY,
+            _ => EMPTY_STR,
+        };
 
         let row = [
             (TABLE_CATALOG, &Value::from(catalog_name)),
@@ -226,6 +320,8 @@ impl InformationSchemaColumnsBuilder {
             (COLUMN_NAME, &Value::from(column_schema.name.as_str())),
             (DATA_TYPE, &Value::from(data_type.as_str())),
             (SEMANTIC_TYPE, &Value::from(semantic_type)),
+            (ORDINAL_POSITION, &Value::from((index + 1) as i64)),
+            (COLUMN_KEY, &Value::from(column_key)),
         ];
 
         if !predicates.eval(&row) {
@@ -236,7 +332,94 @@ impl InformationSchemaColumnsBuilder {
         self.schema_names.push(Some(schema_name));
         self.table_names.push(Some(table_name));
         self.column_names.push(Some(&column_schema.name));
-        self.data_types.push(Some(data_type));
+
+        self.ordinal_positions.push(Some((index + 1) as i64));
+
+        if column_schema.data_type.is_string() {
+            self.character_maximum_lengths.push(Some(MAX_STRING_LENGTH));
+            self.character_octet_lengths.push(Some(MAX_STRING_LENGTH));
+            self.numeric_precisions.push(None);
+            self.numeric_scales.push(None);
+            self.datetime_precisions.push(None);
+            self.character_set_names.push(Some(UTF8_CHARSET_NAME));
+            self.collation_names.push(Some(UTF8_COLLATE_NAME));
+        } else if column_schema.data_type.is_numeric() || column_schema.data_type.is_decimal() {
+            self.character_maximum_lengths.push(None);
+            self.character_octet_lengths.push(None);
+
+            match &column_schema.data_type {
+                ConcreteDataType::Int8(_) | ConcreteDataType::UInt8(_) => {
+                    self.numeric_precisions.push(Some(3));
+                    self.numeric_scales.push(Some(0));
+                }
+                ConcreteDataType::Int16(_) | ConcreteDataType::UInt16(_) => {
+                    self.numeric_precisions.push(Some(5));
+                    self.numeric_scales.push(Some(0));
+                }
+                ConcreteDataType::Int32(_) | ConcreteDataType::UInt32(_) => {
+                    self.numeric_precisions.push(Some(10));
+                    self.numeric_scales.push(Some(0));
+                }
+                ConcreteDataType::Int64(_) => {
+                    self.numeric_precisions.push(Some(19));
+                    self.numeric_scales.push(Some(0));
+                }
+                ConcreteDataType::UInt64(_) => {
+                    self.numeric_precisions.push(Some(20));
+                    self.numeric_scales.push(Some(0));
+                }
+                ConcreteDataType::Float32(_) => {
+                    self.numeric_precisions.push(Some(12));
+                    self.numeric_scales.push(None);
+                }
+                ConcreteDataType::Float64(_) => {
+                    self.numeric_precisions.push(Some(22));
+                    self.numeric_scales.push(None);
+                }
+                ConcreteDataType::Decimal128(decimal_type) => {
+                    self.numeric_precisions
+                        .push(Some(decimal_type.precision() as i64));
+                    self.numeric_scales.push(Some(decimal_type.scale() as i64));
+                }
+                _ => unreachable!(),
+            }
+
+            self.datetime_precisions.push(None);
+            self.character_set_names.push(None);
+            self.collation_names.push(None);
+        } else {
+            self.character_maximum_lengths.push(None);
+            self.character_octet_lengths.push(None);
+            self.numeric_precisions.push(None);
+            self.numeric_scales.push(None);
+
+            match &column_schema.data_type {
+                ConcreteDataType::DateTime(datetime_type) => {
+                    self.datetime_precisions
+                        .push(Some(datetime_type.precision() as i64));
+                }
+                ConcreteDataType::Timestamp(ts_type) => {
+                    self.datetime_precisions
+                        .push(Some(ts_type.precision() as i64));
+                }
+                ConcreteDataType::Time(time_type) => {
+                    self.datetime_precisions
+                        .push(Some(time_type.precision() as i64));
+                }
+                _ => self.datetime_precisions.push(None),
+            }
+
+            self.character_set_names.push(None);
+            self.collation_names.push(None);
+        }
+
+        self.column_keys.push(Some(column_key));
+        self.extras.push(Some(EMPTY_STR));
+        self.privileges.push(Some(DEFAULT_PRIVILEGES));
+        self.generation_expressions.push(Some(EMPTY_STR));
+        self.greptime_data_types
+            .push(Some(&column_schema.data_type.name()));
+        self.data_types.push(Some(&data_type));
         self.semantic_types.push(Some(semantic_type));
         self.column_defaults.push(
             column_schema
@@ -249,7 +432,7 @@ impl InformationSchemaColumnsBuilder {
         } else {
             self.is_nullables.push(Some("No"));
         }
-        self.column_types.push(Some(data_type));
+        self.column_types.push(Some(&data_type));
         self.column_comments
             .push(column_schema.column_comment().map(|x| x.as_ref()));
     }
@@ -260,6 +443,19 @@ impl InformationSchemaColumnsBuilder {
             Arc::new(self.schema_names.finish()),
             Arc::new(self.table_names.finish()),
             Arc::new(self.column_names.finish()),
+            Arc::new(self.ordinal_positions.finish()),
+            Arc::new(self.character_maximum_lengths.finish()),
+            Arc::new(self.character_octet_lengths.finish()),
+            Arc::new(self.numeric_precisions.finish()),
+            Arc::new(self.numeric_scales.finish()),
+            Arc::new(self.datetime_precisions.finish()),
+            Arc::new(self.character_set_names.finish()),
+            Arc::new(self.collation_names.finish()),
+            Arc::new(self.column_keys.finish()),
+            Arc::new(self.extras.finish()),
+            Arc::new(self.privileges.finish()),
+            Arc::new(self.generation_expressions.finish()),
+            Arc::new(self.greptime_data_types.finish()),
             Arc::new(self.data_types.finish()),
             Arc::new(self.semantic_types.finish()),
             Arc::new(self.column_defaults.finish()),

src/datatypes/src/types/datetime_type.rs

@@ -23,10 +23,17 @@ use crate::prelude::{LogicalTypeId, MutableVector, ScalarVectorBuilder, Value, V
 use crate::types::LogicalPrimitiveType;
 use crate::vectors::{DateTimeVector, DateTimeVectorBuilder, PrimitiveVector};
 
+const MILLISECOND_VARIATION: u64 = 3;
 /// Data type for [`DateTime`].
 #[derive(Debug, Default, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
 pub struct DateTimeType;
 
+impl DateTimeType {
+    pub fn precision(&self) -> u64 {
+        MILLISECOND_VARIATION
+    }
+}
+
 impl DataType for DateTimeType {
     fn name(&self) -> String {
         "DateTime".to_string()