Implement Directory Table.

Implement directory table feature in this commit. Directory table is a new
relation which used to organize the unstructured data files in the specified
tablespace. The date files are stored in the specified tablespace while
the tuples recorded the metadata of the data files such as relative_path, md5
size etc. are stored in normal table.

We support local directory table and remote directory table meanwhile. The
local directory table uses the local tablespace while the remote directory
table uses the DFS tablespace which implemented in our enterprise extension.

We support copy binary from to upload file to directory table, directory_table
UDF to get file content, remove_file UDF to remove file from directory table.
What's more, we implement a tool called cbload used to upload file to direcotry
table. Meanwhile, to support DFS directory table, we also import some catalog
tables such as gp_storage_server, gp_storage_user_mapping which are shared in
all databases.

We will illustrage some examples for your convinence of usage as follow.

-- Create an oss_server that points to endpoint:
CREATE STORAGE SERVER oss_server OPTIONS
(protocol 'qingstor', endpoint 'pek3b.qingstor.com', https 'true', virtual_host 'false');

-- Create a user mapping to access oss_server
CREATE STORAGE USER MAPPING FOR CURRENT_USER STORAGE SERVER oss_server OPTIONS
(accesskey 'KGCPPHVCHRDSYFEAWLLC', secretkey '0SJIWiIATh6jOlmAas23q6hOAGBI1BnsnvgJmTs');

-- Create a local tablespace
CREATE TABLESPACE dirtable_spc location '/data/dirtable_spc';

-- Create a local directory table
CREATE DIRECTORY TABLE dirtable TABLESPACE dirtable_spc;

-- Copy binary from directory table
COPY BINARY dirtable FROM '/data/file1.csv' 'file1';

-- Select directory table
SELECT * FROM dirtable;
SELECT * FROM directory_table('dirtable');

-- Remove file from directory table
SELECT remove_file('dirtable', 'file1');

Co-authored-by: Mu Guoqing muguoqing@hashdata.cn
Reviewd-by: Yang Yu yangyu@hashdata.cn
            Yang Jianghua yjhjstz@gmail.com

Author: zhangwenchao-123

contrib/amcheck/verify_heapam.c

@@ -549,10 +549,11 @@ sanity_check_relation(Relation rel)
 {
 	if (rel->rd_rel->relkind != RELKIND_RELATION &&
 		rel->rd_rel->relkind != RELKIND_MATVIEW &&
-		rel->rd_rel->relkind != RELKIND_TOASTVALUE)
+		rel->rd_rel->relkind != RELKIND_TOASTVALUE &&
+		rel->rd_rel->relkind != RELKIND_DIRECTORY_TABLE)
 		ereport(ERROR,
 				(errcode(ERRCODE_WRONG_OBJECT_TYPE),
-				 errmsg("\"%s\" is not a table, materialized view, or TOAST table",
+				 errmsg("\"%s\" is not a table, directory table, materialized view, or TOAST table",
 						RelationGetRelationName(rel))));
 	if (rel->rd_rel->relam != HEAP_TABLE_AM_OID)
 		ereport(ERROR,

contrib/oid2name/oid2name.c

@@ -479,7 +479,7 @@ sql_exec_dumpalltables(PGconn *conn, struct options *opts)
 			 "	LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace "
 			 "	LEFT JOIN pg_catalog.pg_database d ON d.datname = pg_catalog.current_database(),"
 			 "	pg_catalog.pg_tablespace t "
-			 "WHERE relkind IN (" CppAsString2(RELKIND_RELATION) ","
+			 "WHERE relkind IN (" CppAsString2(RELKIND_RELATION) "," CppAsString2(RELKIND_DIRECTORY_TABLE) ","
 			 CppAsString2(RELKIND_MATVIEW) "%s%s) AND "
 			 "	%s"
 			 "		t.oid = CASE"
@@ -553,6 +553,7 @@ sql_exec_searchtables(PGconn *conn, struct options *opts)
 					CppAsString2(RELKIND_MATVIEW) ","
 					CppAsString2(RELKIND_INDEX) ","
 					CppAsString2(RELKIND_SEQUENCE) ","
+					CppAsString2(RELKIND_DIRECTORY_TABLE) ","
 					CppAsString2(RELKIND_TOASTVALUE) ") AND\n"
 					"		t.oid = CASE\n"
 					"			WHEN reltablespace <> 0 THEN reltablespace\n"

contrib/pg_surgery/heap_surgery.c

@@ -374,7 +374,8 @@ sanity_check_relation(Relation rel)
 {
 	if (rel->rd_rel->relkind != RELKIND_RELATION &&
 		rel->rd_rel->relkind != RELKIND_MATVIEW &&
-		rel->rd_rel->relkind != RELKIND_TOASTVALUE)
+		rel->rd_rel->relkind != RELKIND_TOASTVALUE &&
+		rel->rd_rel->relkind != RELKIND_DIRECTORY_TABLE)
 		ereport(ERROR,
 				(errcode(ERRCODE_WRONG_OBJECT_TYPE),
 				 errmsg("\"%s\" is not a table, materialized view, or TOAST table",

contrib/pg_visibility/pg_visibility.c

@@ -778,7 +778,8 @@ check_relation_relkind(Relation rel)
 {
 	if (rel->rd_rel->relkind != RELKIND_RELATION &&
 		rel->rd_rel->relkind != RELKIND_MATVIEW &&
-		rel->rd_rel->relkind != RELKIND_TOASTVALUE)
+		rel->rd_rel->relkind != RELKIND_TOASTVALUE &&
+		rel->rd_rel->relkind != RELKIND_DIRECTORY_TABLE)
 		ereport(ERROR,
 				(errcode(ERRCODE_WRONG_OBJECT_TYPE),
 				 errmsg("\"%s\" is not a table, materialized view, or TOAST table",

contrib/pgstattuple/pgstatapprox.c

@@ -289,10 +289,11 @@ pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo)
 		  rel->rd_rel->relkind == RELKIND_AOBLOCKDIR ||
 		  rel->rd_rel->relkind == RELKIND_AOVISIMAP ||
 		  rel->rd_rel->relkind == RELKIND_MATVIEW ||
-		  rel->rd_rel->relkind == RELKIND_TOASTVALUE))
+		  rel->rd_rel->relkind == RELKIND_TOASTVALUE ||
+		  rel->rd_rel->relkind == RELKIND_DIRECTORY_TABLE))
 		ereport(ERROR,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("\"%s\" is not a table, materialized view, or TOAST table",
+				 errmsg("\"%s\" is not a table, directory table, materialized view, or TOAST table",
 						RelationGetRelationName(rel))));
 
 	if (rel->rd_rel->relam != HEAP_TABLE_AM_OID)

contrib/pgstattuple/pgstatindex.c

@@ -766,9 +766,10 @@ check_relation_relkind(Relation rel)
 		rel->rd_rel->relkind != RELKIND_INDEX &&
 		rel->rd_rel->relkind != RELKIND_MATVIEW &&
 		rel->rd_rel->relkind != RELKIND_SEQUENCE &&
-		rel->rd_rel->relkind != RELKIND_TOASTVALUE)
+		rel->rd_rel->relkind != RELKIND_TOASTVALUE &&
+		rel->rd_rel->relkind != RELKIND_DIRECTORY_TABLE)
 		ereport(ERROR,
 				(errcode(ERRCODE_WRONG_OBJECT_TYPE),
-				 errmsg("\"%s\" is not a table, index, materialized view, sequence, or TOAST table",
+				 errmsg("\"%s\" is not a table, directory table, index, materialized view, sequence, or TOAST table",
 						RelationGetRelationName(rel))));
 }

contrib/pgstattuple/pgstattuple.c

@@ -261,6 +261,7 @@ pgstat_relation(Relation rel, FunctionCallInfo fcinfo)
 		case RELKIND_AOSEGMENTS:
 		case RELKIND_AOBLOCKDIR:
 		case RELKIND_AOVISIMAP:
+		case RELKIND_DIRECTORY_TABLE:
 			return pgstat_heap(rel, fcinfo);
 		case RELKIND_INDEX:
 			switch (rel->rd_rel->relam)

contrib/postgres_fdw/postgres_fdw.c

@@ -5368,6 +5368,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 							   CppAsString2(RELKIND_VIEW) ","
 							   CppAsString2(RELKIND_FOREIGN_TABLE) ","
 							   CppAsString2(RELKIND_MATVIEW) ","
+							   CppAsString2(RELKIND_DIRECTORY_TABLE) ","
 							   CppAsString2(RELKIND_PARTITIONED_TABLE) ") "
 							   "  AND n.nspname = ");
 		deparseStringLiteral(&buf, stmt->remote_schema);

gpMgmt/bin/Makefile

@@ -9,6 +9,7 @@ endif
 
 SUBDIRS = stream gpcheckcat_modules gpconfig_modules gpssh_modules gppylib lib
 SUBDIRS += ifaddrs
+SUBDIRS += cbload
 
 $(recurse)
 

gpMgmt/bin/cbload/Makefile

@@ -0,0 +1,25 @@
+subdir = gpMgmt/bin/cbload
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+.DEFAULT_GOAL := all
+
+export GOPROXY ?= https://proxy.golang.org
+
+all: build
+
+build :
+	go mod download
+	go build -o cbload github.com/cloudberrydb/cbload
+
+clean :
+	rm -f cbload
+
+install: all
+	$(INSTALL_PROGRAM) 'cbload' $(bindir)
+
+uninstall:
+	rm -f $(bindir)/cbload
+
+distclean:
+	rm -f cbload