From 8eae46753837a7f307a6cb3ebd4b014f02434eb0 Mon Sep 17 00:00:00 2001 From: Georgi Ivanov Date: Wed, 16 Oct 2024 14:09:11 +0100 Subject: [PATCH] add a flag to disable transactional support --- CHANGELOG.md | 4 ++ README.md | 2 +- files/startup.sh | 8 ++++ files/update_property.py | 89 ++++++++++++++++++++++++---------------- 4 files changed, 67 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c681af8..2ce6875 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [5.2.2] - 2024-10-16 +### Added +- Added `DISABLE_TRANSACTIONS_SUPPORT` option to disable transactional support in Hive metastore + ## [5.2.1] - 2024-08-23 ### Added - Upgrade yum repos from EMR-5.36.2 (latest EMR 5 version) diff --git a/README.md b/README.md index 5744743..5590620 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi | DATANUCLEUS_CONNECTION_POOL_NAME | No | Connection pool name (HikariCP only). | | DATANUCLEUS_CONNECTION_POOL_CATALOG | No | Connection pool catalog (HikariCP only). | | DATANUCLEUS_CONNECTION_POOL_REGISTER_MBEANS | No | Register MBeans for the connection pool (HikariCP only). | +| DISABLE_TRANSACTIONS_SUPPORT | No | Option to turn disable transactional support in the Hive Metastore. | | DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES | No | `true`/`false` value for hive.metastore.disallow.incompatible.col.type.changes, default `true`. | | ENABLE_GLUESYNC | No | Option to turn on GlueSync Hive Metastore listener. | | ENABLE_HIVE_LOCK_HOUSE_KEEPER | No | Option to turn on Hive Metastore Hive Lock House Keeper. | @@ -68,7 +69,6 @@ For more information please refer to the main [Apiary](https://github.com/Expedi | RANGER_POLICY_MANAGER_URL | No | Ranger admin URL from where policies will be downloaded. | | RANGER_SERVICE_NAME | No | Ranger service name used to configure RangerAuth plugin. | | SNS_ARN | No | The SNS topic ARN to which metadata updates will be - | # Contact diff --git a/files/startup.sh b/files/startup.sh index 725be4f..60d8180 100755 --- a/files/startup.sh +++ b/files/startup.sh @@ -68,6 +68,14 @@ if [ ! -z ${DATANUCLEUS_CONNECTION_POOLING_TYPE} ]; then fi fi +if [[ -n ${DISABLE_TRANSACTIONS_SUPPORT} ]]; then + update_property.py hive.txn.manager org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager /etc/hive/conf/hive-site.xml + update_property.py hive.support.concurrency false /etc/hive/conf/hive-site.xml + update_property.py hive.compactor.initiator.on false /etc/hive/conf/hive-site.xml + update_property.py hive.compactor.worker.threads 0 /etc/hive/conf/hive-site.xml + update_property.py --append hive.conf.restricted.list hive.txn.manager,hive.support.concurrency,hive.compactor.initiator.on,hive.compactor.worker.threads /etc/hive/conf/hive-site.xml +fi + if [[ -n ${DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES} ]]; then update_property.py hive.metastore.disallow.incompatible.col.type.changes "${DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES}" /etc/hive/conf/hive-site.xml fi diff --git a/files/update_property.py b/files/update_property.py index 0f4955f..7f67075 100755 --- a/files/update_property.py +++ b/files/update_property.py @@ -25,42 +25,61 @@ from xml.etree import ElementTree as ET from xml.parsers.expat import ExpatError -def write_properties_to_xml(xml_path, property_name='', property_value=''): - if(os.path.isfile(xml_path)): - try: - xml = ET.parse(xml_path) - except ExpatError: - print "Error while parsing file:" + xml_path - return -1 - property_exists = False - root = xml.getroot() - for child in root.findall('property'): - name = child.find("name").text.strip() - if name == property_name: - property_exists = True - child.find("value").text = property_value - break - if not property_exists: - new_property = ET.SubElement(root, 'property') - ET.SubElement(new_property,'name').text = property_name - ET.SubElement(new_property,'value').text = property_value - - xml.write(xml_path) - return 0 - else: - return -1 +def write_properties_to_xml(xml_path, property_name='', property_value='', append=False): + if os.path.isfile(xml_path): + try: + xml = ET.parse(xml_path) + except ExpatError: + print("Error while parsing file:" + xml_path) + return -1 + + property_exists = False + root = xml.getroot() + + for child in root.findall('property'): + name = child.find("name").text.strip() + if name == property_name: + property_exists = True + current_value = child.find("value").text.strip() + + if append: + # Append the new value to the existing one, separated by commas + if current_value: + new_value = current_value + ',' + property_value + else: + new_value = property_value + else: + new_value = property_value + child.find("value").text = new_value + break + + if not property_exists: + new_property = ET.SubElement(root, 'property') + ET.SubElement(new_property, 'name').text = property_name + ET.SubElement(new_property, 'value').text = property_value + + xml.write(xml_path) + return 0 + else: + print(f"File not found: {xml_path}") + return -1 if __name__ == '__main__': - if(len(sys.argv) > 1): - if(len(sys.argv) > 3): - parameter_name = sys.argv[1] if len(sys.argv) > 1 else None - parameter_value = sys.argv[2] if len(sys.argv) > 2 else None - ranger_admin_site_xml_path = sys.argv[3] if len(sys.argv) > 3 else None - else: - if(len(sys.argv) > 2): - parameter_name = sys.argv[1] if len(sys.argv) > 1 else None - parameter_value = "" - ranger_admin_site_xml_path = sys.argv[2] if len(sys.argv) > 2 else None - write_properties_to_xml(ranger_admin_site_xml_path,parameter_name,parameter_value) + if len(sys.argv) > 1: + append_mode = '--append' in sys.argv # Check if '--append' is passed + if append_mode: + sys.argv.remove('--append') + + if len(sys.argv) > 3: + parameter_name = sys.argv[1] if len(sys.argv) > 1 else None + parameter_value = sys.argv[2] if len(sys.argv) > 2 else None + ranger_admin_site_xml_path = sys.argv[3] if len(sys.argv) > 3 else None + else: + if len(sys.argv) > 2: + parameter_name = sys.argv[1] if len(sys.argv) > 1 else None + parameter_value = "" + file_xml_path = sys.argv[2] if len(sys.argv) > 2 else None + + write_properties_to_xml(file_xml_path, parameter_name, parameter_value, append=append_mode)