diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 94aabe65d433..0703eea9dd91 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -151,8 +151,8 @@ public class HiveIcebergMetaHook implements HiveMetaHook { AlterTableType.ADDCOLS, AlterTableType.REPLACE_COLUMNS, AlterTableType.RENAME_COLUMN, AlterTableType.ADDPROPS, AlterTableType.DROPPROPS, AlterTableType.SETPARTITIONSPEC, AlterTableType.UPDATE_COLUMNS, AlterTableType.RENAME, AlterTableType.EXECUTE, AlterTableType.CREATE_BRANCH, - AlterTableType.CREATE_TAG, AlterTableType.DROP_BRANCH, AlterTableType.DROPPARTITION, AlterTableType.DROP_TAG, - AlterTableType.COMPACT); + AlterTableType.CREATE_TAG, AlterTableType.DROP_BRANCH, AlterTableType.RENAME_BRANCH, AlterTableType.DROPPARTITION, + AlterTableType.DROP_TAG, AlterTableType.COMPACT); private static final List MIGRATION_ALLOWED_SOURCE_FORMATS = ImmutableList.of( FileFormat.PARQUET.name().toLowerCase(), FileFormat.ORC.name().toLowerCase(), diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 147107638ac6..a71fd80a3161 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -1002,6 +1002,11 @@ public void alterTableSnapshotRefOperation(org.apache.hadoop.hive.ql.metadata.Ta (AlterTableSnapshotRefSpec.DropSnapshotRefSpec) alterTableSnapshotRefSpec.getOperationParams(); IcebergBranchExec.dropBranch(icebergTable, dropBranchSpec); break; + case RENAME_BRANCH: + AlterTableSnapshotRefSpec.RenameSnapshotrefSpec renameSnapshotrefSpec = + (AlterTableSnapshotRefSpec.RenameSnapshotrefSpec) alterTableSnapshotRefSpec.getOperationParams(); + IcebergBranchExec.renameBranch(icebergTable, renameSnapshotrefSpec); + break; case DROP_TAG: AlterTableSnapshotRefSpec.DropSnapshotRefSpec dropTagSpec = (AlterTableSnapshotRefSpec.DropSnapshotRefSpec) alterTableSnapshotRefSpec.getOperationParams(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java index 7425ff95c661..9370d7252438 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java @@ -89,4 +89,12 @@ public static void dropBranch(Table table, AlterTableSnapshotRefSpec.DropSnapsho table.manageSnapshots().removeBranch(branchName).commit(); } } + + public static void renameBranch(Table table, AlterTableSnapshotRefSpec.RenameSnapshotrefSpec renameSnapshotrefSpec) { + String sourceBranch = renameSnapshotrefSpec.getSourceBranchName(); + String targetBranch = renameSnapshotrefSpec.getTargetBranchName(); + + LOG.info("Renaming branch {} to {} on iceberg table {}", sourceBranch, targetBranch, table.name()); + table.manageSnapshots().renameBranch(sourceBranch, targetBranch).commit(); + } } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/rename_iceberg_branch.q b/iceberg/iceberg-handler/src/test/queries/positive/rename_iceberg_branch.q new file mode 100644 index 000000000000..e0339bd10c1e --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/rename_iceberg_branch.q @@ -0,0 +1,27 @@ +-- SORT_QUERY_RESULTS +set hive.explain.user=false; +set hive.fetch.task.conversion=more; + +create external table ice01(id int) stored by iceberg stored as orc tblproperties ('format-version'='2'); + +insert into ice01 values (1), (2), (3), (4); + +select * from ice01; + +-- create a branch named soruce +alter table ice01 create branch source; +select * from default.ice01.branch_source; + +-- insert some data to branch +insert into ice01 values (5), (6); +select * from default.ice01.branch_source; + +-- rename the branch +explain alter table ice01 rename branch source to target; +alter table ice01 rename branch source to target; + +select name,type from default.ice01.refs; + +-- read from the renamed branch +select * from default.ice01.branch_target; + diff --git a/iceberg/iceberg-handler/src/test/results/positive/rename_iceberg_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/rename_iceberg_branch.q.out new file mode 100644 index 000000000000..fde01053d5a8 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/rename_iceberg_branch.q.out @@ -0,0 +1,109 @@ +PREHOOK: query: create external table ice01(id int) stored by iceberg stored as orc tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice01 +POSTHOOK: query: create external table ice01(id int) stored by iceberg stored as orc tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice01 +PREHOOK: query: insert into ice01 values (1), (2), (3), (4) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01 +POSTHOOK: query: insert into ice01 values (1), (2), (3), (4) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01 +PREHOOK: query: select * from ice01 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from ice01 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +2 +3 +4 +PREHOOK: query: alter table ice01 create branch source +PREHOOK: type: ALTERTABLE_CREATEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 create branch source +POSTHOOK: type: ALTERTABLE_CREATEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: select * from default.ice01.branch_source +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +2 +3 +4 +PREHOOK: query: insert into ice01 values (5), (6) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice01 +POSTHOOK: query: insert into ice01 values (5), (6) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice01 +PREHOOK: query: select * from default.ice01.branch_source +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +2 +3 +4 +PREHOOK: query: explain alter table ice01 rename branch source to target +PREHOOK: type: ALTERTABLE_RENAMEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: explain alter table ice01 rename branch source to target +POSTHOOK: type: ALTERTABLE_RENAMEBRANCH +POSTHOOK: Input: default@ice01 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + SnapshotRef Operation + table name: default.ice01 + spec: AlterTableSnapshotRefSpec{operationType=RENAME_BRANCH, operationParams=RenameSnapshotrefSpec{sourceBranch=source, targetBranch=target}} + +PREHOOK: query: alter table ice01 rename branch source to target +PREHOOK: type: ALTERTABLE_RENAMEBRANCH +PREHOOK: Input: default@ice01 +POSTHOOK: query: alter table ice01 rename branch source to target +POSTHOOK: type: ALTERTABLE_RENAMEBRANCH +POSTHOOK: Input: default@ice01 +PREHOOK: query: select name,type from default.ice01.refs +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select name,type from default.ice01.refs +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +main BRANCH +target BRANCH +PREHOOK: query: select * from default.ice01.branch_target +PREHOOK: type: QUERY +PREHOOK: Input: default@ice01 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from default.ice01.branch_target +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice01 +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +2 +3 +4 diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g index 873a28eb05cd..a628c4364ed5 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g @@ -80,6 +80,7 @@ alterTableStatementSuffix | alterStatementSuffixCreateTag | alterStatementSuffixDropTag | alterStatementSuffixConvert + | alterStatementSuffixRenameBranch ; alterTblPartitionStatementSuffix[boolean partition] @@ -505,6 +506,13 @@ alterStatementSuffixExecute -> ^(TOK_ALTERTABLE_EXECUTE KW_ORPHAN_FILES $timestamp?) ; +alterStatementSuffixRenameBranch +@init { gParent.pushMsg("alter table rename branch", state); } +@after { gParent.popMsg(state); } + : KW_RENAME KW_BRANCH sourceBranch=identifier KW_TO targetBranch=identifier + -> ^(TOK_ALTERTABLE_RENAME_BRANCH $sourceBranch $targetBranch) + ; + alterStatementSuffixDropBranch @init { gParent.pushMsg("alter table drop branch (if exists) branchName", state); } @after { gParent.popMsg(state); } diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index d9cdac0c0d19..9c2c0c6b3027 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -221,6 +221,7 @@ TOK_ALTERTABLE_SETPARTSPEC; TOK_ALTERTABLE_EXECUTE; TOK_ALTERTABLE_CREATE_BRANCH; TOK_ALTERTABLE_DROP_BRANCH; +TOK_ALTERTABLE_RENAME_BRANCH; TOK_ALTERTABLE_CREATE_TAG; TOK_ALTERTABLE_DROP_TAG; TOK_RETAIN; diff --git a/pom.xml b/pom.xml index 2fe6cd7ab46e..307da3930205 100644 --- a/pom.xml +++ b/pom.xml @@ -217,7 +217,7 @@ 2.0.1 2.9.0 3.0.11 - 1.1.0-incubating + 1.2.0 4.0.3 1.1.0.Final 1.0.1 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java index 7da882c641bb..cc78ba6317dc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java @@ -42,6 +42,7 @@ public enum AlterTableType { EXECUTE("execute"), CREATE_BRANCH("create branch"), DROP_BRANCH("drop branch"), + RENAME_BRANCH("rename branch"), CREATE_TAG("create tag"), DROP_TAG("drop tag"), // constraint diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/rename/AlterTableRenameSnapshotRefAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/rename/AlterTableRenameSnapshotRefAnalyzer.java new file mode 100644 index 000000000000..89cc11fb0f60 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/rename/AlterTableRenameSnapshotRefAnalyzer.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.ddl.table.snapshotref.branch.rename; + +import java.util.Map; + +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.ddl.DDLWork; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; +import org.apache.hadoop.hive.ql.ddl.table.snapshotref.AlterTableSnapshotRefDesc; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERTABLE_RENAME_BRANCH) +public class AlterTableRenameSnapshotRefAnalyzer extends AbstractAlterTableAnalyzer { + + protected AlterTableType alterTableType; + + public AlterTableRenameSnapshotRefAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + alterTableType = AlterTableType.RENAME_BRANCH; + } + + @Override + protected void analyzeCommand(TableName tableName, Map partitionSpec, ASTNode command) + throws SemanticException { + Table table = getTable(tableName); + DDLUtils.validateTableIsIceberg(table); + inputs.add(new ReadEntity(table)); + validateAlterTableType(table, alterTableType, false); + String sourceBranch = command.getChild(0).getText(); + String targetBranch = command.getChild(1).getText(); + + AlterTableSnapshotRefSpec.RenameSnapshotrefSpec renameSnapshotrefSpec = + new AlterTableSnapshotRefSpec.RenameSnapshotrefSpec(sourceBranch, targetBranch); + AlterTableSnapshotRefSpec alterTableSnapshotRefSpec = + new AlterTableSnapshotRefSpec(alterTableType, renameSnapshotrefSpec); + AbstractAlterTableDesc alterTableDesc = + new AlterTableSnapshotRefDesc(alterTableType, tableName, alterTableSnapshotRefSpec); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTableDesc))); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index e97b56b4dca6..28f35c4a15fe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -616,6 +616,7 @@ public final class FunctionRegistry { system.registerGenericUDF("array_union", GenericUDFArrayUnion.class); system.registerGenericUDF("array_remove", GenericUDFArrayRemove.class); system.registerGenericUDF("array_position", GenericUDFArrayPosition.class); + system.registerGenericUDF("array_append", GenericUDFArrayAppend.class); system.registerGenericUDF("deserialize", GenericUDFDeserialize.class); system.registerGenericUDF("sentences", GenericUDFSentences.class); system.registerGenericUDF("map_keys", GenericUDFMapKeys.class); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java index 1300f5f5a1bc..004086528300 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java @@ -122,4 +122,29 @@ public String toString() { return MoreObjects.toStringHelper(this).add("refName", refName).add("ifExists", ifExists).toString(); } } + + public static class RenameSnapshotrefSpec { + + private final String sourceBranch; + private final String targetBranch; + + public String getSourceBranchName() { + return sourceBranch; + } + + public String getTargetBranchName() { + return targetBranch; + } + + public RenameSnapshotrefSpec(String sourceBranch, String targetBranch) { + this.sourceBranch = sourceBranch; + this.targetBranch = targetBranch; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("sourceBranch", sourceBranch).add("targetBranch", targetBranch) + .toString(); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index 020453d682b8..d616912659a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -80,6 +80,7 @@ public enum HiveOperation { ALTERTABLE_CREATEBRANCH("ALTERTABLE_CREATEBRANCH", HiveParser.TOK_ALTERTABLE_CREATE_BRANCH, null, null), ALTERTABLE_CREATETAG("ALTERTABLE_CREATETAG", HiveParser.TOK_ALTERTABLE_CREATE_TAG, null, null), ALTERTABLE_DROPBRANCH("ALTERTABLE_DROPBRANCH", HiveParser.TOK_ALTERTABLE_DROP_BRANCH, null, null), + ALTERTABLE_RENAMEBRANCH("ALTERTABLE_RENAMEBRANCH", HiveParser.TOK_ALTERTABLE_RENAME_BRANCH, null, null), ALTERTABLE_DROPTAG("ALTERTABLE_DROPTAG", HiveParser.TOK_ALTERTABLE_DROP_TAG, null, null), ALTERTABLE_CONVERT("ALTERTABLE_CONVERT", HiveParser.TOK_ALTERTABLE_CONVERT, null, null), ALTERTABLE_SERIALIZER("ALTERTABLE_SERIALIZER", HiveParser.TOK_ALTERTABLE_SERIALIZER, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index 68d008eeda38..e09cc8c78084 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -141,6 +141,7 @@ public enum HiveOperationType { ALTERTABLE_UPDATECOLUMNS, ALTERTABLE_CREATEBRANCH, ALTERTABLE_DROPBRANCH, + ALTERTABLE_RENAMEBRANCH, ALTERTABLE_CREATETAG, ALTERTABLE_DROPTAG, SHOW_COMPACTIONS, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java index 4cb7a57cadeb..b200522f5a9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java @@ -246,6 +246,8 @@ public HivePrivilegeObjectType getObjectType() { PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_DROPBRANCH, PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); + op2Priv.put(HiveOperationType.ALTERTABLE_RENAMEBRANCH, + PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_DROPTAG, PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayAppend.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayAppend.java new file mode 100644 index 000000000000..0a41ee56efa6 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayAppend.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * GenericUDFArrayAppend. + */ +@Description(name = "array_append", value = "_FUNC_(array, element) - Returns an array appended by element.", + extended = "Example:\n" + " > SELECT _FUNC_(array(1,3,4), 2) FROM src;\n" + " [1,3,4,2]") +public class GenericUDFArrayAppend extends AbstractGenericUDFArrayBase { + private static final String FUNC_NAME = "ARRAY_APPEND"; + private static final int ELEMENT_IDX = 1; + + public GenericUDFArrayAppend() { + super(FUNC_NAME, 2, 2, ObjectInspector.Category.LIST); + } + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + ObjectInspector defaultOI = super.initialize(arguments); + checkValueAndListElementTypes(arrayElementOI, FUNC_NAME, arguments[ELEMENT_IDX], ELEMENT_IDX); + return defaultOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object array = arguments[ARRAY_IDX].get(); + Object value = arguments[ELEMENT_IDX].get(); + int arrayLength = arrayOI.getListLength(array); + if (arrayLength == 0) { + return Collections.emptyList(); + } else if (arrayLength < 0) { + return null; + } + + List resultArray = new ArrayList<>(((ListObjectInspector) argumentOIs[ARRAY_IDX]).getList(array)); + resultArray.add(value); + return resultArray.stream().map(o -> converter.convert(o)).collect(Collectors.toList()); + } +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFArrayAppend.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFArrayAppend.java new file mode 100644 index 000000000000..2dc8fb328a52 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFArrayAppend.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Arrays.asList; + +public class TestGenericUDFArrayAppend { + private final GenericUDFArrayAppend udf = new GenericUDFArrayAppend(); + + @Test + public void testPrimitive() throws HiveException { + ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector( + PrimitiveObjectInspectorFactory.writableIntObjectInspector), + PrimitiveObjectInspectorFactory.writableIntObjectInspector }; + udf.initialize(inputOIs); + + Object i1 = new IntWritable(3); + Object i2 = new IntWritable(1); + Object i3 = new IntWritable(2); + Object i4 = new IntWritable(1); + Object i5 = new IntWritable(5); + + runAndVerify(asList(i1, i2, i3, i4), i5, asList(i1, i2, i3, i4, i5)); + i1 = new FloatWritable(3.3f); + i2 = new FloatWritable(1.1f); + i3 = new FloatWritable(3.3f); + i4 = new FloatWritable(2.20f); + i5 = new FloatWritable(5.20f); + runAndVerify(asList(i1, i2, i3, i4), i5, asList(i1, i2, i3, i4, i5)); + runAndVerify(asList(i1, i2, i3, i4), null, asList(i1, i2, i3, i4, null)); //Test null element + } + + @Test + public void testList() throws HiveException { + ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector( + ObjectInspectorFactory.getStandardListObjectInspector( + PrimitiveObjectInspectorFactory.writableStringObjectInspector)), + ObjectInspectorFactory.getStandardListObjectInspector( + PrimitiveObjectInspectorFactory.writableStringObjectInspector) }; + udf.initialize(inputOIs); + + Object i1 = asList(new Text("aa1"), new Text("dd"), new Text("cc"), new Text("bb")); + Object i2 = asList(new Text("aa2"), new Text("cc"), new Text("ba"), new Text("dd")); + Object i3 = asList(new Text("aa3"), new Text("cc"), new Text("dd"), new Text("ee"), new Text("bb")); + Object i4 = asList(new Text("aa4"), new Text("cc"), new Text("ddd"), new Text("bb")); + runAndVerify(asList(i1, i2, i2, i3, i4), i4, asList(i1, i2, i2, i3, i4, i4)); + } + + @Test + public void testStruct() throws HiveException { + ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector( + ObjectInspectorFactory.getStandardStructObjectInspector(asList("f1", "f2", "f3", "f4"), + asList(PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, + PrimitiveObjectInspectorFactory.writableDateObjectInspector, + ObjectInspectorFactory.getStandardListObjectInspector( + PrimitiveObjectInspectorFactory.writableIntObjectInspector)))), + ObjectInspectorFactory.getStandardStructObjectInspector(asList("f1", "f2", "f3", "f4"), + asList(PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, + PrimitiveObjectInspectorFactory.writableDateObjectInspector, + ObjectInspectorFactory.getStandardListObjectInspector( + PrimitiveObjectInspectorFactory.writableIntObjectInspector))) }; + udf.initialize(inputOIs); + + Object i1 = asList(new Text("a"), new DoubleWritable(3.1415), new DateWritableV2(Date.of(2015, 5, 26)), + asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(4))); + + Object i2 = asList(new Text("b"), new DoubleWritable(3.14), new DateWritableV2(Date.of(2015, 5, 26)), + asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(4))); + + Object i3 = asList(new Text("a"), new DoubleWritable(3.1415), new DateWritableV2(Date.of(2015, 5, 25)), + asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(5))); + + Object i4 = asList(new Text("a"), new DoubleWritable(3.1415), new DateWritableV2(Date.of(2015, 5, 25)), + asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(4))); + + runAndVerify(asList(i1, i3, i2, i3, i4), i2, asList(i1, i3, i2, i3, i4, i2)); + } + + @Test + public void testMap() throws HiveException { + ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector( + ObjectInspectorFactory.getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableIntObjectInspector)), + ObjectInspectorFactory.getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.writableStringObjectInspector, + PrimitiveObjectInspectorFactory.writableIntObjectInspector) }; + udf.initialize(inputOIs); + + Map m1 = new HashMap(); + m1.put(new Text("a"), new IntWritable(4)); + m1.put(new Text("b"), new IntWritable(3)); + m1.put(new Text("c"), new IntWritable(1)); + m1.put(new Text("d"), new IntWritable(2)); + + Map m2 = new HashMap(); + m2.put(new Text("d"), new IntWritable(4)); + m2.put(new Text("b"), new IntWritable(3)); + m2.put(new Text("a"), new IntWritable(1)); + m2.put(new Text("c"), new IntWritable(2)); + + Map m3 = new HashMap(); + m3.put(new Text("d"), new IntWritable(4)); + m3.put(new Text("b"), new IntWritable(3)); + m3.put(new Text("a"), new IntWritable(1)); + + runAndVerify(asList(m1, m3, m2, m3, m1), m2, asList(m1, m3, m2, m3, m1, m2)); + } + + private void runAndVerify(List actual, Object element, List expected) throws HiveException { + GenericUDF.DeferredJavaObject[] args = + { new GenericUDF.DeferredJavaObject(actual), new GenericUDF.DeferredJavaObject(element) }; + List result = (List) udf.evaluate(args); + if (expected == null) { + Assert.assertNull(result); + } else { + Assert.assertArrayEquals("Check content", expected.toArray(), result.toArray()); + } + } +} diff --git a/ql/src/test/queries/clientnegative/udf_array_append_1.q b/ql/src/test/queries/clientnegative/udf_array_append_1.q new file mode 100644 index 000000000000..5cee7106ddc2 --- /dev/null +++ b/ql/src/test/queries/clientnegative/udf_array_append_1.q @@ -0,0 +1 @@ +SELECT array_append(3, 3); \ No newline at end of file diff --git a/ql/src/test/queries/clientnegative/udf_array_append_2.q b/ql/src/test/queries/clientnegative/udf_array_append_2.q new file mode 100644 index 000000000000..21d0375c3d20 --- /dev/null +++ b/ql/src/test/queries/clientnegative/udf_array_append_2.q @@ -0,0 +1 @@ +SELECT array_append(array(3), array(3)); \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/udf_array_append.q b/ql/src/test/queries/clientpositive/udf_array_append.q new file mode 100644 index 000000000000..a4c2c67fc947 --- /dev/null +++ b/ql/src/test/queries/clientpositive/udf_array_append.q @@ -0,0 +1,44 @@ +--! qt:dataset:src + +-- SORT_QUERY_RESULTS + +set hive.fetch.task.conversion=more; + +DESCRIBE FUNCTION array_append; +DESCRIBE FUNCTION EXTENDED array_append; + +-- evaluates function for array of primitives +SELECT array_append(array(1, 2, 3, null,3,4), 3); + +SELECT array_append(array(1.12, 2.23, 3.34, null,1.11,1.12,2.9),1.12); + +SELECT array(1,2,3),array_append(array(1, 2, 3),3); + +SELECT array(1,2,3),array_append(array(1, 2, 3),5); + +SELECT array_append(array(1, 2, 3), CAST(null AS int)); + +SELECT array_append(array(1.1234567890, 2.234567890, 3.34567890, null, 3.3456789, 2.234567,1.1234567890),1.1234567890); + +SELECT array_append(array(11234567890, 2234567890, 334567890, null, 11234567890, 2234567890, 334567890, null),11234567890); + +SELECT array_append(array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null,array("e","a","b","c","d")),array("a","b","c","d")); + +SELECT array_append(array("aa","bb","cc"),"cc"); + +# handle null array cases + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/test_null_array; + +dfs -copyFromLocal ../../data/files/test_null_array.csv ${system:test.tmp.dir}/test_null_array/; + +create external table test_null_array (id string, value Array) ROW FORMAT DELIMITED + FIELDS TERMINATED BY ':' collection items terminated by ',' location '${system:test.tmp.dir}/test_null_array'; + +select id,value from test_null_array; + +select id, array_append(value,id) from test_null_array; + +select value, array_append(value,id) from test_null_array; + +dfs -rm -r ${system:test.tmp.dir}/test_null_array; \ No newline at end of file diff --git a/ql/src/test/results/clientnegative/udf_array_append_1.q.out b/ql/src/test/results/clientnegative/udf_array_append_1.q.out new file mode 100644 index 000000000000..02edd70792e3 --- /dev/null +++ b/ql/src/test/results/clientnegative/udf_array_append_1.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10016]: Line 1:20 Argument type mismatch '3': "array" expected at function ARRAY_APPEND, but "int" is found diff --git a/ql/src/test/results/clientnegative/udf_array_append_2.q.out b/ql/src/test/results/clientnegative/udf_array_append_2.q.out new file mode 100644 index 000000000000..eeeaa1a1a8a6 --- /dev/null +++ b/ql/src/test/results/clientnegative/udf_array_append_2.q.out @@ -0,0 +1 @@ +FAILED: SemanticException [Error 10016]: Line 1:30 Argument type mismatch '3': int type element is expected at function ARRAY_APPEND(array,int), but array is found diff --git a/ql/src/test/results/clientpositive/llap/show_functions.q.out b/ql/src/test/results/clientpositive/llap/show_functions.q.out index 24105d060102..b764c89e94d2 100644 --- a/ql/src/test/results/clientpositive/llap/show_functions.q.out +++ b/ql/src/test/results/clientpositive/llap/show_functions.q.out @@ -46,6 +46,7 @@ aes_encrypt and approx_distinct array +array_append array_contains array_distinct array_except @@ -675,6 +676,7 @@ aes_encrypt and approx_distinct array +array_append array_contains array_distinct array_except diff --git a/ql/src/test/results/clientpositive/llap/udf_array_append.q.out b/ql/src/test/results/clientpositive/llap/udf_array_append.q.out new file mode 100644 index 000000000000..35694a7cacd0 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/udf_array_append.q.out @@ -0,0 +1,141 @@ +PREHOOK: query: DESCRIBE FUNCTION array_append +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION array_append +POSTHOOK: type: DESCFUNCTION +array_append(array, element) - Returns an array appended by element. +PREHOOK: query: DESCRIBE FUNCTION EXTENDED array_append +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED array_append +POSTHOOK: type: DESCFUNCTION +array_append(array, element) - Returns an array appended by element. +Example: + > SELECT array_append(array(1,3,4), 2) FROM src; + [1,3,4,2] +Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDFArrayAppend +Function type:BUILTIN +PREHOOK: query: SELECT array_append(array(1, 2, 3, null,3,4), 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_append(array(1, 2, 3, null,3,4), 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[1,2,3,null,3,4,3] +PREHOOK: query: SELECT array_append(array(1.12, 2.23, 3.34, null,1.11,1.12,2.9),1.12) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_append(array(1.12, 2.23, 3.34, null,1.11,1.12,2.9),1.12) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[1.12,2.23,3.34,null,1.11,1.12,2.9,1.12] +PREHOOK: query: SELECT array(1,2,3),array_append(array(1, 2, 3),3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array(1,2,3),array_append(array(1, 2, 3),3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[1,2,3] [1,2,3,3] +PREHOOK: query: SELECT array(1,2,3),array_append(array(1, 2, 3),5) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array(1,2,3),array_append(array(1, 2, 3),5) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[1,2,3] [1,2,3,5] +PREHOOK: query: SELECT array_append(array(1, 2, 3), CAST(null AS int)) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_append(array(1, 2, 3), CAST(null AS int)) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[1,2,3,null] +PREHOOK: query: SELECT array_append(array(1.1234567890, 2.234567890, 3.34567890, null, 3.3456789, 2.234567,1.1234567890),1.1234567890) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_append(array(1.1234567890, 2.234567890, 3.34567890, null, 3.3456789, 2.234567,1.1234567890),1.1234567890) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[1.123456789,2.23456789,3.3456789,null,3.3456789,2.234567,1.123456789,1.123456789] +PREHOOK: query: SELECT array_append(array(11234567890, 2234567890, 334567890, null, 11234567890, 2234567890, 334567890, null),11234567890) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_append(array(11234567890, 2234567890, 334567890, null, 11234567890, 2234567890, 334567890, null),11234567890) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[11234567890,2234567890,334567890,null,11234567890,2234567890,334567890,null,11234567890] +PREHOOK: query: SELECT array_append(array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null,array("e","a","b","c","d")),array("a","b","c","d")) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_append(array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null,array("e","a","b","c","d")),array("a","b","c","d")) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +[["a","b","c","d"],["a","b","c","d"],["a","b","c","d","e"],null,["e","a","b","c","d"],["a","b","c","d"]] +PREHOOK: query: SELECT array_append(array("aa","bb","cc"),"cc") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT array_append(array("aa","bb","cc"),"cc") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +["aa","bb","cc","cc"] +PREHOOK: query: create external table test_null_array (id string, value Array) ROW FORMAT DELIMITED +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_null_array +POSTHOOK: query: create external table test_null_array (id string, value Array) ROW FORMAT DELIMITED +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_null_array +PREHOOK: query: select id,value from test_null_array +PREHOOK: type: QUERY +PREHOOK: Input: default@test_null_array +#### A masked pattern was here #### +POSTHOOK: query: select id,value from test_null_array +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_null_array +#### A masked pattern was here #### +1 [] +2 ["NULL"] +3 ["null","null"] +PREHOOK: query: select id, array_append(value,id) from test_null_array +PREHOOK: type: QUERY +PREHOOK: Input: default@test_null_array +#### A masked pattern was here #### +POSTHOOK: query: select id, array_append(value,id) from test_null_array +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_null_array +#### A masked pattern was here #### +1 [] +2 ["NULL","2"] +3 ["null","null","3"] +PREHOOK: query: select value, array_append(value,id) from test_null_array +PREHOOK: type: QUERY +PREHOOK: Input: default@test_null_array +#### A masked pattern was here #### +POSTHOOK: query: select value, array_append(value,id) from test_null_array +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_null_array +#### A masked pattern was here #### +["NULL"] ["NULL","2"] +["null","null"] ["null","null","3"] +[] [] diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 12d2481a5c4d..b124873b11be 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -62,7 +62,7 @@ 3.12.0 1.1.3 2.9.0 - 1.1.0-incubating + 1.2.0 5.2.8 5.2.10 3.2.0-release