From 2c899bd799fbd26488ff2dc7ce8a434f9ed757a5 Mon Sep 17 00:00:00 2001 From: eelke Date: Mon, 6 Feb 2023 20:03:57 +0100 Subject: [PATCH] Generate PARTITIONED BY SQL for partitioned tables. Expressions not yet supported. --- pglab/OpenDatabase.cpp | 2 +- pglablib/catalog/PgAttributeContainer.cpp | 2 +- pglablib/catalog/PgClass.cpp | 146 ++++++++++++++---- pglablib/catalog/PgClass.h | 41 +++++ pglablib/catalog/PgClassContainer.cpp | 70 ++++++++- pglablib/catalog/PgContainer.h | 10 ++ pgsql/Pgsql_Col.h | 5 + .../notes/partition-by-414aefaf17009287.yaml | 8 + 8 files changed, 243 insertions(+), 41 deletions(-) create mode 100644 releasenotes/notes/partition-by-414aefaf17009287.yaml diff --git a/pglab/OpenDatabase.cpp b/pglab/OpenDatabase.cpp index dac9160..71323ac 100644 --- a/pglab/OpenDatabase.cpp +++ b/pglab/OpenDatabase.cpp @@ -12,7 +12,7 @@ OpenDatabase::OpenDatabaseSPtr OpenDatabase::createOpenDatabase(const Connection odb->Init(); return odb; } - catch (const Pgsql::PgException &ex) { + catch (const std::exception &ex) { throw OpenDatabaseException(ex.what()); } } diff --git a/pglablib/catalog/PgAttributeContainer.cpp b/pglablib/catalog/PgAttributeContainer.cpp index 8ea91d7..edbdbcf 100644 --- a/pglablib/catalog/PgAttributeContainer.cpp +++ b/pglablib/catalog/PgAttributeContainer.cpp @@ -23,7 +23,7 @@ SELECT attrelid, attname, atttypid, attstattarget, LEFT JOIN pg_attrdef AS def ON attrelid=adrelid AND attnum=adnum LEFT JOIN (pg_depend JOIN pg_class cs ON classid='pg_class'::regclass AND objid=cs.oid AND cs.relkind='S') ON refobjid=att.attrelid AND refobjsubid=att.attnum LEFT JOIN pg_namespace ns ON ns.oid=cs.relnamespace - LEFT JOIN pg_catalog.pg_description AS d ON (objoid=attrelid AND d.objsubid=attnum))__"; + LEFT JOIN pg_catalog.pg_description AS d ON (objoid=attrelid AND d.objsubid=attnum) )__"; return q; } diff --git a/pglablib/catalog/PgClass.cpp b/pglablib/catalog/PgClass.cpp index 7f41cbb..15751ff 100644 --- a/pglablib/catalog/PgClass.cpp +++ b/pglablib/catalog/PgClass.cpp @@ -1,4 +1,5 @@ #include "PgClass.h" +#include "PgAttribute.h" #include "PgAttributeContainer.h" #include "PgClassContainer.h" #include "PgDatabaseCatalog.h" @@ -7,6 +8,7 @@ #include #include "SqlFormattingUtils.h" #include +#include void operator<<(RelPersistence &s, const Pgsql::Value &v) @@ -33,41 +35,60 @@ void operator<<(RelKind &s, const Pgsql::Value &v) const char *c = v.c_str(); switch (*c) { - case 'r': - s = RelKind::Table; - break; - case 'i': - s = RelKind::Index; - break; - case 'S': - s = RelKind::Sequence; - break; - case 'v': - s = RelKind::View; - break; - case 'm': - s = RelKind::MaterializedView; - break; - case 'c': - s = RelKind::Composite; - break; - case 't': - s = RelKind::Toast; - break; - case 'f': - s = RelKind::ForeignTable; - break; - case 'p': - s = RelKind::PartitionedTable; - break; - case 'I': - s = RelKind::PartitionedIndex; - break; - default: - throw std::runtime_error("Unknown RelKind"); + case 'r': + s = RelKind::Table; + break; + case 'i': + s = RelKind::Index; + break; + case 'S': + s = RelKind::Sequence; + break; + case 'v': + s = RelKind::View; + break; + case 'm': + s = RelKind::MaterializedView; + break; + case 'c': + s = RelKind::Composite; + break; + case 't': + s = RelKind::Toast; + break; + case 'f': + s = RelKind::ForeignTable; + break; + case 'p': + s = RelKind::PartitionedTable; + break; + case 'I': + s = RelKind::PartitionedIndex; + break; + default: + throw std::runtime_error("Unknown RelKind"); } } +void operator<<(PartitioningStrategy &s, const Pgsql::Value &v) +{ + const char *c = v.c_str(); + switch (*c) + { + case 'h': + s = PartitioningStrategy::Hash; + break; + case 'l': + s = PartitioningStrategy::List; + break; + case 'r': + s = PartitioningStrategy::Range; + break; + default: + throw std::runtime_error("Unknown PartitioningStrategy"); + } +} + QString PgClass::createSql() const { if (createSqlCache.isEmpty()) @@ -75,6 +96,7 @@ QString PgClass::createSql() const switch (kind) { case RelKind::Table: + case RelKind::PartitionedTable: createSqlCache = createTableSql(); break; case RelKind::View: @@ -118,7 +140,7 @@ QString PgClass::ddlTypeName() const { switch (kind) { - case RelKind::PartitionedTable: return "PARTITIONED TABLE"; + case RelKind::PartitionedTable: return "TABLE"; default: return PgNamespaceObject::ddlTypeName(); @@ -145,7 +167,9 @@ QString PgClass::createTableSql() const else sql += generateBodySql(false); + sql += generateInheritsSql() + % partitionBySql() % generateTablespaceSql() % ";\n"; return sql; @@ -238,6 +262,49 @@ QString PgClass::generateInheritsSql() const return sql; } +QString PgClass::partitionBySql() const +{ + if (kind != RelKind::PartitionedTable) + return {}; + + QString sql = "\nPARTITION BY " % PartitionStrategyKeyword(partitionedTable.strategy); + sql += partitionKeySql(); + return sql; +} + +QString PgClass::partitionKeySql() const +{ + QString result; + result += "("; + auto keyItem = partitionedTable.keyColumns.begin(); + if (keyItem != partitionedTable.keyColumns.end()) + { + result += partitionKeyItemSql(*keyItem); + for (++keyItem; keyItem != partitionedTable.keyColumns.end(); ++keyItem) + result += ", " % partitionKeyItemSql(*keyItem); + } + result += ")"; + return result; +} + +QString PgClass::partitionKeyItemSql( + const PartitioningKeyItem &keyItem +) const +{ + if (keyItem.attNum == 0) + return "\"\""; // TODO add expression support for now use this to prevent a crash here because column 0 does not exist + + const PgAttribute *col = catalog().attributes()->findIf( + [this, &keyItem] (const auto &att) + { + return att.relid == oid() && att.num == keyItem.attNum; + } + ); + + assert(col != nullptr); + return quoteIdent(col->name); +} + QString PgClass::getPartitionOfName() const { auto parents = catalog().inherits()->getParentsOf(oid()); @@ -274,4 +341,15 @@ QString PgClass::createViewSql() const return sql; } - +QString PartitionStrategyKeyword(PartitioningStrategy ps) +{ + switch (ps) { + case PartitioningStrategy::Hash: + return "HASH"; + case PartitioningStrategy::List: + return "LIST"; + case PartitioningStrategy::Range: + return "RANGE"; + } + throw std::runtime_error("Unknown PartitioningStrategy"); +} diff --git a/pglablib/catalog/PgClass.h b/pglablib/catalog/PgClass.h index d35e91c..53ff6db 100644 --- a/pglablib/catalog/PgClass.h +++ b/pglablib/catalog/PgClass.h @@ -5,6 +5,9 @@ #include "PgNamespaceObject.h" #include #include +#include + +class PgAttribute; enum class RelPersistence { Permanent, // p @@ -30,6 +33,37 @@ enum class RelKind void operator<<(RelKind &s, const Pgsql::Value &v); +enum class PartitioningStrategy +{ + Hash, // h + List, // l + Range // r +}; + +void operator<<(PartitioningStrategy &s, const Pgsql::Value &v); + +QString PartitionStrategyKeyword(PartitioningStrategy ps); + +class PartitioningKeyItem +{ +public: + int16_t attNum; + Oid opClass; + Oid collation; + // expre nodetree + QString expression; // pg_get_expr(pg_node_tree, relation_oid) +}; +using PartitioningKeyItems = boost::container::small_vector; + +class PgPartitionedTable +{ +public: + PartitioningStrategy strategy; + Oid defaultPartition; + + PartitioningKeyItems keyColumns; +}; + class PgClass: public PgNamespaceObject { public: @@ -51,6 +85,7 @@ public: std::vector options; QString viewdef; QString partitionBoundaries; + PgPartitionedTable partitionedTable; // ignore if RelKind != PartitionedTable using PgNamespaceObject::PgNamespaceObject; @@ -59,6 +94,7 @@ public: QString typeName() const override; QString aclAllPattern() const override; + protected: virtual QString ddlTypeName() const override; @@ -68,6 +104,11 @@ private: QString createTableSql() const; QString generateBodySql(bool isPartition) const; QString generateInheritsSql() const; + QString partitionBySql() const; + QString partitionKeySql() const; + QString partitionKeyItemSql( + const PartitioningKeyItem &keyItem + ) const; QString getPartitionOfName() const; QString generateTablespaceSql() const; QString createViewSql() const; diff --git a/pglablib/catalog/PgClassContainer.cpp b/pglablib/catalog/PgClassContainer.cpp index 426b390..f1a3360 100644 --- a/pglablib/catalog/PgClassContainer.cpp +++ b/pglablib/catalog/PgClassContainer.cpp @@ -14,19 +14,55 @@ std::string PgClassContainer::getLoadQuery() const " reloptions, d.description, " " relacl, pg_get_viewdef(pg_class.oid)"; - if (lessThenVersion(120000)) + if (lessThenVersion(120000)) q += ", relhasoids "; if (minimumVersion(100000)) - q += ", pg_get_expr(relpartbound, oid)"; // partition specification + q += + ", pg_get_expr(relpartbound, oid)" + ", partstrat, partnatts, partattrs, partclass, partcollation"; // TODO: , partexprs"; + if (minimumVersion(110000)) + q += ", partdefid"; - q += + // pg_get_expr must be called on each element in partexprs + + q += "\nFROM pg_catalog.pg_class \n" - " LEFT JOIN pg_catalog.pg_description AS d ON (objoid=pg_class.oid AND objsubid=0) \n" + " LEFT JOIN pg_catalog.pg_description AS d ON (objoid=pg_class.oid AND objsubid=0) \n"; + + if (minimumVersion(100000)) + q += " LEFT JOIN pg_partitioned_table AS pt ON (partrelid=pg_class.oid) \n"; + + q += "WHERE relkind IN ('r', 'i', 'p', 'I', 'v', 'm', 'f')"; return q; } +namespace { + + class PartitionedTableKeyItemsBuilder { + public: + int16_t attCount; + std::vector attNums; + std::vector attOpClass; + std::vector attCollation; + + PartitioningKeyItems Build() + { + PartitioningKeyItems result(attCount); + for (int attIdx = 0; attIdx < attCount; ++attIdx) + { + auto& item = result[attIdx]; + item.attNum = attNums[attIdx]; + item.opClass = attOpClass[attIdx]; + item.collation = attCollation[attIdx]; + } + return result; + } + }; + +} + PgClass PgClassContainer::loadElem(const Pgsql::Row &row) { Pgsql::Col col(row); @@ -35,7 +71,7 @@ PgClass PgClassContainer::loadElem(const Pgsql::Row &row) Oid schema_oid = col.nextValue(); PgClass v(m_catalog, class_oid, name, schema_oid); - Oid owner ; + Oid owner; col >> v.type >> v.oftype >> owner >> v.am >> v.filenode >> v.tablespace >> v.pages_est >> v.tuples_est >> v.toastrelid >> v.isshared >> v.persistence @@ -55,8 +91,32 @@ PgClass PgClassContainer::loadElem(const Pgsql::Row &row) if (lessThenVersion(120000)) col >> v.hasoids; + PgPartitionedTable &pt = v.partitionedTable; if (minimumVersion(100000)) + { + PartitionedTableKeyItemsBuilder kibuilder; + col >> v.partitionBoundaries; + auto strategy = col.nextValue(); + if (strategy.null()) + { + int s = minimumVersion(110000) ? 5 : 4; + col.skip(s); + } + else + { + pt.strategy << strategy; + col >> kibuilder.attCount; + col.getAsVector(std::back_inserter(kibuilder.attNums)); + col.getAsVector(std::back_inserter(kibuilder.attOpClass)); + col.getAsVector(std::back_inserter(kibuilder.attCollation)); + + pt.keyColumns = kibuilder.Build(); + if (minimumVersion(110000)) + col >> pt.defaultPartition; + } + } + return v; } diff --git a/pglablib/catalog/PgContainer.h b/pglablib/catalog/PgContainer.h index 85d940a..0d8737b 100644 --- a/pglablib/catalog/PgContainer.h +++ b/pglablib/catalog/PgContainer.h @@ -4,6 +4,7 @@ #include "Pgsql_declare.h" #include "Pgsql_Result.h" #include +#include #include #include #include @@ -68,6 +69,15 @@ public: return nullptr; } + const T* findIf(std::function func) const + { + auto findResult = std::find_if(m_container.begin(), m_container.end(), func); + if (findResult != m_container.end()) + return &*findResult; + + return nullptr; + } + const T* getByName(const QString &name) const { auto find_res = std::find(m_container.begin(), m_container.end(), name); diff --git a/pgsql/Pgsql_Col.h b/pgsql/Pgsql_Col.h index 7c105c8..1d68afc 100644 --- a/pgsql/Pgsql_Col.h +++ b/pgsql/Pgsql_Col.h @@ -19,6 +19,11 @@ namespace Pgsql { return row.get(++col); } + void skip(int n = 1) + { + col += n; + } + template Col& getAsArray(I insert_iter, NullHandling nullhandling = NullHandling::Throw) { diff --git a/releasenotes/notes/partition-by-414aefaf17009287.yaml b/releasenotes/notes/partition-by-414aefaf17009287.yaml new file mode 100644 index 0000000..425b5d0 --- /dev/null +++ b/releasenotes/notes/partition-by-414aefaf17009287.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Limited support generating SQL for partitioned tables. It should be able to + generate SQL for list, hash and range partitioning involving columns only. +issues: + - | + It can yet generate correct SQL for a table partitioned by an expression.