From ee32cad303cb571cdbc55bb66c52cdea6818616d Mon Sep 17 00:00:00 2001 From: BUGKICKER <517441250@qq.com> Date: Tue, 9 Apr 2024 18:13:52 +0800 Subject: [PATCH] Support Hive 3.x --- .../geaflow-dsl-connector-hive/pom.xml | 13 ++-- .../connector/hive/adapter/Hive3Adapter.java | 63 +++++++++++++++++++ .../hive/adapter/HiveVersionAdapters.java | 13 ++++ .../dsl/connector/hive/BaseHiveTest.java | 2 + 4 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/Hive3Adapter.java diff --git a/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/pom.xml b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/pom.xml index ec71a9c04..7316081ee 100644 --- a/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/pom.xml +++ b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/pom.xml @@ -27,8 +27,8 @@ geaflow-dsl-connector-hive - 2.3.6 - 2.7.4 + 3.1.2 + 3.1.0 4.0.0 0.9.10 7.6.0.v20120127 @@ -251,10 +251,6 @@ org.apache.calcite calcite-core - - org.apache.calcite - calcite-druid - org.apache.calcite.avatica avatica @@ -366,5 +362,10 @@ + + hadoop-hdfs + org.apache.hadoop + ${hadoop.version} + diff --git a/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/Hive3Adapter.java b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/Hive3Adapter.java new file mode 100644 index 000000000..1414730bd --- /dev/null +++ b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/Hive3Adapter.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 AntGroup CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +package com.antgroup.geaflow.dsl.connector.hive.adapter; + +import com.antgroup.geaflow.dsl.common.exception.GeaFlowDSLException; +import java.lang.reflect.Method; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Table; + +public class Hive3Adapter implements HiveVersionAdapter { + + private final String version; + private static final String METHOD_NAME_GET_PROXY = "getProxy"; + private static final String METHOD_NAME_GET_TABLE_META_DATA = "getTableMetadata"; + + public Hive3Adapter(String version) { + this.version = version; + } + + @Override + public String version() { + return version; + } + + @Override + public IMetaStoreClient createMetaSoreClient(HiveConf hiveConf) { + try { + Method method = RetryingMetaStoreClient.class + .getMethod(METHOD_NAME_GET_PROXY, Configuration.class, Boolean.TYPE); + return (IMetaStoreClient) method.invoke(null, hiveConf, true); + } catch (Exception ex) { + throw new RuntimeException("Failed to create Hive Metastore client", ex); + } + } + + @Override + public Properties getTableMetadata(Table table) { + try { + Class metaStoreUtilsClass = Class.forName("org.apache.hadoop.hive.metastore.utils.MetaStoreUtils"); + Method method = + metaStoreUtilsClass.getMethod(METHOD_NAME_GET_TABLE_META_DATA, Table.class); + return (Properties) method.invoke(null, table); + } catch (Exception e) { + throw new GeaFlowDSLException(e); + } + } +} diff --git a/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/HiveVersionAdapters.java b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/HiveVersionAdapters.java index fd80927e3..d52092a6c 100644 --- a/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/HiveVersionAdapters.java +++ b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/main/java/com/antgroup/geaflow/dsl/connector/hive/adapter/HiveVersionAdapters.java @@ -37,6 +37,14 @@ public class HiveVersionAdapters { public static final String HIVE_239 = "2.3.9"; + public static final String HIVE_300 = "3.0.0"; + + public static final String HIVE_310 = "3.1.0"; + + public static final String HIVE_311 = "3.1.1"; + + public static final String HIVE_312 = "3.1.2"; + public static HiveVersionAdapter get() { Package myPackage = HiveVersionAnnotation.class.getPackage(); HiveVersionAnnotation version = myPackage.getAnnotation(HiveVersionAnnotation.class); @@ -51,6 +59,11 @@ public static HiveVersionAdapter get() { case HIVE_237: case HIVE_239: return new Hive23Adapter(version.version()); + case HIVE_300: + case HIVE_310: + case HIVE_311: + case HIVE_312: + return new Hive3Adapter(version.version()); default: throw new GeaFlowDSLException("Hive version: {} is not supported.", version.version()); } diff --git a/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/test/java/com/antgroup/geaflow/dsl/connector/hive/BaseHiveTest.java b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/test/java/com/antgroup/geaflow/dsl/connector/hive/BaseHiveTest.java index 936ff517b..5057cf2ec 100644 --- a/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/test/java/com/antgroup/geaflow/dsl/connector/hive/BaseHiveTest.java +++ b/geaflow/geaflow-dsl/geaflow-dsl-connector/geaflow-dsl-connector-hive/src/test/java/com/antgroup/geaflow/dsl/connector/hive/BaseHiveTest.java @@ -74,6 +74,8 @@ private HiveConf createHiveConf(Configuration conf, String hiveLocation) { conf.set("hive.metastore.schema.verification", "false"); conf.set("datanucleus.autoCreateSchema", "true"); conf.set("datanucleus.fixedDatastore", "false"); + conf.set("datanucleus.schema.autoCreateAll", "true"); + conf.set("hive.stats.autogather","false"); String scratchDir = FileUtil.concatPath(hiveLocation, "scratch"); conf.set(ConfVars.SCRATCHDIR.varname, scratchDir);