From 082974d42db6726a997e1c0901618e1956c085f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 31 Mar 2021 18:00:51 +0300 Subject: [PATCH 001/106] Initial test pipeline --- .github/workflows/tool-tests.yml | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/tool-tests.yml diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml new file mode 100644 index 00000000..12039a45 --- /dev/null +++ b/.github/workflows/tool-tests.yml @@ -0,0 +1,37 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the develop branch + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Runs a single command using the runners shell + - name: Run a one-line script + run: echo Hello, world! + + # Runs a set of commands using the runners shell + - name: Run a multi-line script + run: | + export GH_TOKEN=$(GH_TOKEN) + export BASE_PATH=$(pwd) + python -m unittest /vagrant/release/tools/python/tests/test_update_package_properties.py From e332ab546d02508f2ed3fa733072c6bc4810c362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 31 Mar 2021 21:22:04 +0300 Subject: [PATCH 002/106] Fix path problem --- .github/workflows/tool-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 12039a45..17a07328 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -32,6 +32,7 @@ jobs: # Runs a set of commands using the runners shell - name: Run a multi-line script run: | + ls -la export GH_TOKEN=$(GH_TOKEN) export BASE_PATH=$(pwd) - python -m unittest /vagrant/release/tools/python/tests/test_update_package_properties.py + python -m unittest ./python/tests/test_update_package_properties.py From 831ca888fec75cc57a2a9e10bdb929a6fda64114 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 31 Mar 2021 22:30:29 +0300 Subject: [PATCH 003/106] Add update_package_properties and its unit tests --- .github/workflows/tool-tests.yml | 1 + bash/execute_unit_tests.sh | 1 + bash/execute_update_properties.sh | 12 + python/debian/changelog | 1767 ++++++++++++++++ python/pkgvars | 6 + python/publish-packages.py | 103 + python/requirements.txt | 73 + python/templates/pkgvars.tmpl | 6 + python/templates/project.spec.tmpl | 82 + python/templates/showpersons.txt | 3 + python/tests/files/citus.spec | 318 +++ python/tests/files/debian.changelog.refer | 1767 ++++++++++++++++ python/tests/files/pkgvars | 6 + .../verify/debian_changelog_with_10.0.3.txt | 1802 +++++++++++++++++ .../verify/expected_changelog_10.0.3.txt | 33 + .../verify/expected_debian_latest_v10.0.3.txt | 34 + .../verify/rpm_latest_changelog_reference.txt | 2 + .../tests/test_update_package_properties.py | 154 ++ python/tests/test_utils.py | 10 + python/tests/to_be_deleted/actual.txt | 315 +++ python/tests/to_be_deleted/expected | 318 +++ python/update_package_properties.py | 319 +++ python/upload_to_package_cloud.py | 110 + 23 files changed, 7242 insertions(+) create mode 100644 bash/execute_unit_tests.sh create mode 100644 bash/execute_update_properties.sh create mode 100644 python/debian/changelog create mode 100644 python/pkgvars create mode 100644 python/publish-packages.py create mode 100644 python/requirements.txt create mode 100644 python/templates/pkgvars.tmpl create mode 100644 python/templates/project.spec.tmpl create mode 100644 python/templates/showpersons.txt create mode 100644 python/tests/files/citus.spec create mode 100644 python/tests/files/debian.changelog.refer create mode 100644 python/tests/files/pkgvars create mode 100644 python/tests/files/verify/debian_changelog_with_10.0.3.txt create mode 100644 python/tests/files/verify/expected_changelog_10.0.3.txt create mode 100644 python/tests/files/verify/expected_debian_latest_v10.0.3.txt create mode 100644 python/tests/files/verify/rpm_latest_changelog_reference.txt create mode 100644 python/tests/test_update_package_properties.py create mode 100644 python/tests/test_utils.py create mode 100644 python/tests/to_be_deleted/actual.txt create mode 100644 python/tests/to_be_deleted/expected create mode 100644 python/update_package_properties.py create mode 100644 python/upload_to_package_cloud.py diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 17a07328..10894a3a 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -33,6 +33,7 @@ jobs: - name: Run a multi-line script run: | ls -la + pip install -r requirements.txt export GH_TOKEN=$(GH_TOKEN) export BASE_PATH=$(pwd) python -m unittest ./python/tests/test_update_package_properties.py diff --git a/bash/execute_unit_tests.sh b/bash/execute_unit_tests.sh new file mode 100644 index 00000000..a9bf588e --- /dev/null +++ b/bash/execute_unit_tests.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/bash/execute_update_properties.sh b/bash/execute_update_properties.sh new file mode 100644 index 00000000..aab50c8b --- /dev/null +++ b/bash/execute_update_properties.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +git clone https://github.com/citusdata/packaging.git +cd packaging +git checkout all-citus +python ../../python/update_package_properties.py --gh_token "${GITHUB_TOKEN}" --prj_name "${PRJ_NAME}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ + --fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S &z')" --exec-path "$(pwd)" &>2 + +#cd .. +#echo "Removing Packaging directory..." +#rm -r packaging +#echo "Packaging directory Removed" diff --git a/python/debian/changelog b/python/debian/changelog new file mode 100644 index 00000000..58d1e481 --- /dev/null +++ b/python/debian/changelog @@ -0,0 +1,1767 @@ +citus (10.0.2.citus-1) stable; urgency=low + + * Adds a configure flag to enforce security + + * Fixes a bug due to cross join without target list + + * Fixes a bug with UNION ALL on PG 13 + + * Fixes a compatibility issue with pg_audit in utility calls + + * Fixes insert query with CTEs/sublinks/subqueries etc + + * Grants SELECT permission on citus_tables view to public + + * Grants SELECT permission on columnar metadata tables to public + + * Improves citus_update_table_statistics and provides distributed deadlock + detection + + * Preserves colocation with procedures in alter_distributed_table + + * Prevents using alter_columnar_table_set and alter_columnar_table_reset + on a columnar table not owned by the user + + * Removes limits around long table names + + -- Gurkan Indibay Thu, 4 Mar 2021 2:46:54 +0000 + +citus (9.5.2.citus-1) stable; urgency=low + + * Fixes distributed deadlock detection being blocked by metadata sync + + * Prevents segfaults when SAVEPOINT handling cannot recover from connection + failures + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Wed, 27 Jan 2021 11:25:38 +0000 + +citus (9.4.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause router queries with local tables to be pushed + down + + * Fixes a segfault in connection management due to invalid connection hash + entries + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Tue, 5 Jan 2021 14:58:56 +0000 + +citus (9.5.1.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that could cause excessive memory consumption when a partition is + created + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + * Fixes a segfault in connection management due to invalid connection hash + entries + + -- Onur Tirtir Wed, 2 Dec 2020 14:28:44 +0000 + +citus (9.4.3.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + -- Onur Tirtir Tue, 24 Nov 2020 11:17:57 +0000 + +citus (9.5.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 13 + + * Removes the task-tracker executor + + * Introduces citus local tables + + * Introduces undistribute_table UDF to convert tables back to postgres tables + + * Adds support for EXPLAIN (ANALYZE) EXECUTE and EXPLAIN EXECUTE + + * Adds support for EXPLAIN (ANALYZE, WAL) for PG13 + + * Sorts the output of EXPLAIN (ANALYZE) by execution duration. + + * Adds support for CREATE TABLE ... USING table_access_method + + * Adds support for WITH TIES option in SELECT and INSERT SELECT queries + + * Avoids taking multi-shard locks on workers + + * Enforces citus.max_shared_pool_size config in COPY queries + + * Enables custom aggregates with multiple parameters to be executed on workers + + * Enforces citus.max_intermediate_result_size in local execution + + * Improves cost estimation of INSERT SELECT plans + + * Introduces delegation of procedures that read from reference tables + + * Prevents pull-push execution for simple pushdownable subqueries + + * Improves error message when creating a foreign key to a local table + + * Makes citus_prepare_pg_upgrade idempotent by dropping transition tables + + * Disallows ON TRUE outer joins with reference & distributed tables when + reference table is outer relation to avoid incorrect results + + * Disallows field indirection in INSERT/UPDATE queries to avoid incorrect + results + + * Disallows volatile functions in UPDATE subqueries to avoid incorrect results + + * Fixes CREATE INDEX CONCURRENTLY crash with local execution + + * Fixes citus_finish_pg_upgrade to drop all backup tables + + * Fixes a bug that cause failures when RECURSIVE VIEW joined reference table + + * Fixes DROP SEQUENCE failures when metadata syncing is enabled + + * Fixes a bug that caused CREATE TABLE with CHECK constraint to fail + + * Fixes a bug that could cause VACUUM to deadlock + + * Fixes master_update_node failure when no background worker slots are available + + * Fixes a bug that caused replica identity to not be propagated on shard repair + + * Fixes a bug that could cause crashes after connection timeouts + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug that could cause deadlocks on CREATE INDEX + + * Fixes a bug with genetic query optimization in outer joins + + * Fixes a crash when aggregating empty tables + + * Fixes a crash with inserting domain constrained composite types + + * Fixes a crash with multi-row & router INSERT's in local execution + + * Fixes a possibility of doing temporary file cleanup more than once + + * Fixes incorrect setting of join related fields + + * Fixes memory issues around deparsing index commands + + * Fixes reference table access tracking for sequential execution + + * Fixes removal of a single node with only reference tables + + * Fixes sending commands to coordinator when it is added as a worker + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlock + + -- Onur Tirtir Wed, 11 Nov 2020 15:00:27 +0000 + +citus (9.4.2.citus-1) stable; urgency=low + + * Fixes a bug that could lead to multiple maintenance daemons + + * Fixes an issue preventing views in reference table modifications + + -- Onur Tirtir Thu, 22 Oct 2020 8:53:44 +0000 + +citus (9.4.1.citus-1) stable; urgency=low + + * Fixes EXPLAIN ANALYZE output truncation + + * Fixes a deadlock during transaction recovery + + -- Onur Tirtir Wed, 30 Sep 2020 9:33:46 +0000 + +citus (9.4.0.citus-1) stable; urgency=low + + * Improves COPY by honoring max_adaptive_executor_pool_size config + + * Adds support for insert into local table select from distributed table + + * Adds support to partially push down tdigest aggregates + + * Adds support for receiving binary encoded results from workers using + citus.enable_binary_protocol + + * Enables joins between local tables and CTEs + + * Adds showing query text in EXPLAIN output when explain verbose is true + + * Adds support for showing CTE statistics in EXPLAIN ANALYZE + + * Adds support for showing amount of data received in EXPLAIN ANALYZE + + * Introduces downgrade paths in migration scripts + + * Avoids returning incorrect results when changing roles in a transaction + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes a bug that occurs when coordinator is added as a worker node + + * Fixes a crash because of overflow in partition id with certain compile flags + + * Fixes a crash that may happen if no worker nodes are added + + * Fixes a crash that occurs when inserting implicitly coerced constants + + * Fixes a crash when aggregating empty tables + + * Fixes a memory leak in subtransaction memory handling + + * Fixes crash when using rollback to savepoint after cancellation of DML + + * Fixes deparsing for queries with anonymous column references + + * Fixes distribution of composite types failing to include typemods + + * Fixes explain analyze on adaptive executor repartitions + + * Fixes possible error throwing in abort handle + + * Fixes segfault when evaluating func calls with default params on coordinator + + * Fixes several EXPLAIN ANALYZE issues + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlocks + + * Reports correct INSERT/SELECT method in EXPLAIN + + * Disallows triggers on citus tables + + -- Onur Tirtir Tue, 28 Jul 2020 13:22:31 +0000 + +citus (9.3.5.citus-1) stable; urgency=low + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a crash because of overflow in partition id with certain compile flags + + -- Onur Tirtir Mon, 27 Jul 2020 7:28:18 +0000 + +citus (9.3.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes deparsing for queries with anonymous column references + + -- Onur Tirtir Wed, 22 Jul 2020 9:00:01 +0000 + +citus (9.3.3.citus-1) stable; urgency=low + + * Fixes a memory leak in subtransaction memory handling + + -- Onur Tirtir Mon, 13 Jul 2020 8:47:40 +0000 + +citus (9.3.0.citus-1) stable; urgency=low + + * Adds max_shared_pool_size to control number of connections across sessions + + * Adds support for window functions on coordinator + + * Improves shard pruning logic to understand OR-conditions + + * Prevents using an extra connection for intermediate result multi-casts + + * Adds propagation of ALTER ROLE .. SET statements + + * Adds update_distributed_table_colocation UDF to update colocation of tables + + * Introduces a UDF to truncate local data after distributing a table + + * Adds support for creating temp schemas in parallel + + * Adds support for evaluation of nextval in the target list on coordinator + + * Adds support for local execution of COPY/TRUNCATE/DROP/DDL commands + + * Adds support for local execution of shard creation + + * Uses local execution in a transaction block + + * Adds support for querying distributed table sizes concurrently + + * Allows master_copy_shard_placement to replicate placements to new nodes + + * Allows table type to be used in target list + + * Avoids having multiple maintenance daemons active for a single database + + * Defers reference table replication to shard creation time + + * Enables joins between local tables and reference tables in transaction blocks + + * Ignores pruned target list entries in coordinator plan + + * Improves SIGTERM handling of maintenance daemon + + * Increases the default of citus.node_connection_timeout to 30 seconds + + * Fixes a bug that occurs when creating remote tasks in local execution + + * Fixes a bug that causes some DML queries containing aggregates to fail + + * Fixes a bug that could cause failures in queries with subqueries or CTEs + + * Fixes a bug that may cause some connection failures to throw errors + + * Fixes a bug which caused queries with SRFs and function evalution to fail + + * Fixes a bug with generated columns when executing COPY dist_table TO file + + * Fixes a crash when using non-constant limit clauses + + * Fixes a failure when composite types used in prepared statements + + * Fixes a possible segfault when dropping dist. table in a transaction block + + * Fixes a possible segfault when non-pushdownable aggs are solely used in HAVING + + * Fixes a segfault when executing queries using GROUPING + + * Fixes an error when using LEFT JOIN with GROUP BY on primary key + + * Fixes an issue with distributing tables having generated cols not at the end + + * Fixes automatic SSL permission issue when using "initdb --allow-group-access" + + * Fixes errors which could occur when subqueries are parameters to aggregates + + * Fixes possible issues by invalidating the plan cache in master_update_node + + * Fixes timing issues which could be caused by changing system clock + + -- Onur Tirtir Thu, 7 May 2020 15:11:25 +0000 + +citus (9.2.4.citus-1) stable; urgency=low + + * Fixes a release problem in 9.2.3 + + -- Onur Tirtir Tue, 31 Mar 2020 08:06:59 +0000 + +citus (9.2.3.citus-1) stable; urgency=low + + * Do not use C functions that have been banned by Microsoft + + * Fixes a bug that causes wrong results with complex outer joins + + * Fixes issues found using static analysis + + * Fixes left join shard pruning in pushdown planner + + * Fixes possibility for segmentation fault in internal aggregate functions + + * Fixes possible segfault when non pushdownable aggregates are used in HAVING + + * Improves correctness of planning subqueries in HAVING + + * Prevents using old connections for security if citus.node_conninfo changed + + * Uses Microsoft approved cipher string for default TLS setup + + -- Onur Tirtir Thu, 26 Mar 2020 8:22:48 +0000 + +citus (9.0.2.citus-1) stable; urgency=low + + * Fixes build errors on EL/OL 6 based distros + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Disallows distributed function creation when replication_model is `statement` + + -- Onur Tirtir Fri, 6 Mar 2020 14:10:16 +0000 + +citus (9.2.2.citus-1) stable; urgency=low + + * Fixes a bug that caused some prepared stmts with function calls to fail + + * Fixes a bug that caused some prepared stmts with composite types to fail + + * Fixes a bug that caused missing subplan results in workers + + * Improves performance of re-partition joins + + -- Onur Tirtir Fri, 6 Mar 2020 07:14:20 +0000 + +citus (9.2.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes if distribution key is NULL + + -- Onur Tirtir Fri, 14 Feb 2020 11:51:09 +0000 + +citus (9.2.0.citus-1) stable; urgency=low + + * Adds support for INSERT...SELECT queries with re-partitioning + + * Adds citus.coordinator_aggregation_strategy to support more aggregates + + * Adds caching of local plans on shards for Citus MX + + * Adds compatibility support for dist. object infrastructure from old versions + + * Adds defering shard-pruning for fast-path router queries to execution + + * Adds propagation of GRANT ... ON SCHEMA queries + + * Adds support for CTE pushdown via CTE inlining in distributed planning + + * Adds support for ALTER TABLE ... SET SCHEMA propagation. + + * Adds support for DROP ROUTINE & ALTER ROUTINE commands + + * Adds support for any inner join on a reference table + + * Changes citus.log_remote_commands level to NOTICE + + * Disallows marking ref. table shards unhealthy in the presence of savepoints + + * Disallows placing new shards with shards in TO_DELETE state + + * Enables local execution of queries that do not need any data access + + * Fixes Makefile trying to cleanup PG directory during install + + * Fixes a bug causing errors when planning a query with multiple subqueries + + * Fixes a possible deadlock that could happen during shard moves + + * Fixes a problem when adding a new node due to tables referenced in func body + + * Fixes an issue that could cause joins with reference tables to be slow + + * Fixes cached metadata for shard is inconsistent issue + + * Fixes inserting multiple composite types as partition key in VALUES + + * Fixes unnecessary repartition on joins with more than 4 tables + + * Prevents wrong results for replicated partitioned tables after failure + + * Restricts LIMIT approximation for non-commutative aggregates + + -- Onur Tirtir Wed, 10 Feb 2020 8:48:00 +0000 + +citus (9.1.1.citus-1) stable; urgency=low + + * Fixes a bug causing SQL-executing UDFs to crash when passing in DDL + + * Fixes a bug that caused column_to_column_name to crash for invalid input + + * Fixes a bug that caused inserts into local tables w/ dist. subqueries to crash + + * Fixes a bug that caused some noop DML statements to fail + + * Fixes a bug that prevents dropping reference table columns + + * Fixes a crash in IN (.., NULL) queries + + * Fixes a crash when calling a distributed function from PL/pgSQL + + * Fixes an issue that caused CTEs to sometimes leak connections + + * Fixes strange errors in DML with unreachable sublinks + + * Prevents statements in SQL functions to run outside of a transaction + + -- Onur Tirtir Wed, 18 Dec 2019 14:32:42 +0000 + +citus (9.1.0.citus-1) stable; urgency=low + + * Adds extensions to distributed object propagation infrastructure + + * Adds support for ALTER ROLE propagation + + * Adds support for aggregates in create_distributed_function + + * Adds support for expressions in reference joins + + * Adds support for returning RECORD in multi-shard queries + + * Adds support for simple IN subqueries on unique cols in repartition joins + + * Adds support for subqueries in HAVING clauses + + * Automatically distributes unary aggs w/ combinefunc and non-internal stype + + * Disallows distributed func creation when replication_model is 'statement' + + * Drops support for deprecated real-time and router executors + + * Fixes a bug in local execution that could cause missing rows in RETURNING + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Fixes a bug that caused other CREATE EXTENSION commands to take longer + + * Fixes a bug that prevented REFRESH MATERIALIZED VIEW + + * Fixes a bug when view is used in modify statements + + * Fixes a memory leak in adaptive executor when query returns many columns + + * Fixes underflow init of default values in worker extended op node creation + + * Fixes potential segfault in standard_planner inlining functions + + * Fixes an issue that caused failures in RHEL 6 builds + + * Fixes queries with repartition joins and group by unique column + + * Improves CTE/Subquery performance by pruning intermediate rslt broadcasting + + * Removes citus.worker_list_file GUC + + * Revokes usage from the citus schema from public + + -- Onur Tirtir Thu, 28 Nov 2019 15:11:05 +0000 + +citus (9.0.1.citus-1) stable; urgency=low + + * Fixes a memory leak in the executor + + * Revokes usage from the citus schema from public + + -- Hanefi Onaldi Wed, 30 Oct 2019 8:53:22 +0000 + +citus (9.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 12 + + * Adds UDFs to help with PostgreSQL upgrades + + * Distributes types to worker nodes + + * Introduces create_distributed_function UDF + + * Introduces local query execution for Citus MX + + * Implements infrastructure for routing CALL to MX workers + + * Implements infrastructure for routing SELECT function() to MX workers + + * Adds support for foreign key constraints between reference tables + + * Adds a feature flag to turn off CREATE TYPE propagation + + * Adds option citus.single_shard_commit_protocol + + * Adds support for EXPLAIN SUMMARY + + * Adds support for GENERATE ALWAYS AS STORED + + * Adds support for serial and smallserial in MX mode + + * Adds support for anon composite types on the target list in router queries + + * Avoids race condition between create_reference_table & master_add_node + + * Fixes a bug in schemas of distributed sequence definitions + + * Fixes a bug that caused run_command_on_colocated_placements to fail + + * Fixes a bug that leads to various issues when a connection is lost + + * Fixes a schema leak on CREATE INDEX statement + + * Fixes assert failure in bare SELECT FROM reference table FOR UPDATE in MX + + * Makes master_update_node MX compatible + + * Prevents pg_dist_colocation from multiple records for reference tables + + * Prevents segfault in worker_partition_protocol edgecase + + * Propagates ALTER FUNCTION statements for distributed functions + + * Propagates CREATE OR REPLACE FUNCTION for distributed functions + + * Propagates REINDEX on tables & indexes + + * Provides a GUC to turn of the new dependency propagation functionality + + * Uses 2PC in adaptive executor when dealing with replication factors above 1 + + -- Hanefi Onaldi Tue, 15 Oct 2019 16:54:50 +0000 + +citus (8.3.2.citus-1) stable; urgency=low + + * Fixes performance issues by skipping unnecessary relation access recordings + + -- Hanefi Onaldi Fri, 9 Aug 2019 11:15:57 +0000 + +citus (8.3.1.citus-1) stable; urgency=low + + * Improves Adaptive Executor performance + + -- Hanefi Onaldi Mon, 29 Jul 2019 10:25:50 +0000 + +citus (8.3.0.citus-1) stable; urgency=low + + * Adds a new distributed executor: Adaptive Executor + + * citus.enable_statistics_collection defaults to off (opt-in) + + * Adds support for CTEs in router planner for modification queries + + * Adds support for propagating SET LOCAL at xact start + + * Adds option to force master_update_node during failover + + * Deprecates master_modify_multiple_shards + + * Improves round robin logic on router queries + + * Creates all distributed schemas as superuser on a separate connection + + * Makes COPY adapt to connection use behaviour of previous commands + + * Replaces SESSION_LIFESPAN with configurable no. of connections at xact end + + * Propagates ALTER FOREIGN TABLE commands to workers + + * Don't schedule tasks on inactive nodes + + * Makes DROP/VALIDATE CONSTRAINT tolerant of ambiguous shard extension + + * Fixes an issue with subquery map merge jobs as non-root + + * Fixes null pointers caused by partial initialization of ConnParamsHashEntry + + * Fixes errors caused by joins with shadowed aliases + + * Fixes a regression in outer joining subqueries introduced in 8.2.0 + + * Fixes a crash that can occur under high memory load + + * Fixes a bug that selects wrong worker when using round-robin assignment + + * Fixes savepoint rollback after multi-shard modify/copy failure + + * Fixes bad foreign constraint name search + + * Fixes a bug that prevents stack size to be adjusted + + -- Hanefi Onaldi Wed, 10 Jul 2019 15:19:02 +0000 + +citus (8.2.2.citus-1) stable; urgency=low + + * Fixes a bug in outer joins wrapped in subqueries + + -- Burak Velioglu Wed, 12 Jun 2019 8:45:08 +0000 + +citus (8.2.1.citus-1) stable; urgency=low + + * Fixes a bug that prevents stack size to be adjusted + + -- Burak Velioglu Wed, 3 Apr 2019 20:56:47 +0000 + +citus (8.1.2.citus-1) stable; urgency=low + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Fixes a bug that prevents stack size to be adjusted + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + -- Burak Velioglu Wed, 3 Apr 2019 20:34:46 +0000 + +citus (8.2.0.citus-1) stable; urgency=low + + * Removes support and code for PostgreSQL 9.6 + + * Enable more outer joins with reference tables + + * Execute CREATE INDEX CONCURRENTLY in parallel + + * Treat functions as transaction blocks + + * Add support for column aliases on join clauses + + * Skip standard_planner() for trivial queries + + * Added support for function calls in joins + + * Round-robin task assignment policy relies on local transaction id + + * Relax subquery union pushdown restrictions for reference tables + + * Speed-up run_command_on_shards() + + * Address some memory issues in connection config + + * Restrict visibility of get_*_active_transactions functions to pg_monitor + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Queries with only intermediate results do not rely on task assignment policy + + * Finish connection establishment in parallel for multiple connections + + * Fixes a bug related to pruning shards using a coerced value + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + * Fixes a bug that could lead to infinite recursion during recursive planning + + * Fixes a bug that could prevent planning full outer joins with using clause + + * Fixes a bug that could lead to memory leak on citus_relation_size + + * Fixes a problem that could cause segmentation fault with recursive planning + + * Switch CI solution to CircleCI + + -- Burak Velioglu Fri, 29 Mar 2019 07:36:09 +0000 + +citus (8.0.3.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Wed, 9 Jan 2019 9:50:07 +0000 + +citus (8.1.1.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Mon, 7 Jan 2019 16:26:13 +0000 + +citus (8.1.0.citus-1) stable; urgency=low + + * Turns on ssl by default for new installations of citus + + * Restricts SSL Ciphers to TLS1.2 and above + + * Adds support for INSERT INTO SELECT..ON CONFLICT/RETURNING via coordinator + + * Adds support for round-robin task assignment for queries to reference tables + + * Adds support for SQL tasks using worker_execute_sql_task UDF with task-tracker + + * Adds support for VALIDATE CONSTRAINT queries + + * Adds support for disabling hash aggregate with HLL + + * Adds user ID suffix to intermediate files generated by task-tracker + + * Only allow transmit from pgsql_job_cache directory + + * Disallows GROUPING SET clauses in subqueries + + * Removes restriction on user-defined group ID in node addition functions + + * Relaxes multi-shard modify locks when enable_deadlock_prevention is disabled + + * Improves security in task-tracker protocol + + * Improves permission checks in internal DROP TABLE functions + + * Improves permission checks in cluster management functions + + * Cleans up UDFs and fixes permission checks + + * Fixes crashes caused by stack size increase under high memory load + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Tue, 18 Dec 2018 15:12:45 +0000 + +citus (8.0.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Thu, 13 Dec 2018 13:56:44 +0000 + +citus (7.5.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Wed, 12 Dec 2018 11:45:24 +0000 + +citus (8.0.1.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 11:38:47 +0000 + +citus (7.5.3.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 10:52:20 +0000 + +citus (7.5.2.citus-1) stable; urgency=low + + * Fixes inconsistent metadata error when shard metadata caching get interrupted + + * Fixes a bug that could cause memory leak + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Wed, 14 Nov 2018 20:42:16 +0000 + +citus (8.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 11 + + * Adds support for applying DML operations on reference tables from MX nodes + + * Adds distributed locking to truncated MX tables + + * Adds support for running TRUNCATE command from MX worker nodes + + * Adds views to provide insight about the distributed transactions + + * Adds support for TABLESAMPLE in router queries + + * Adds support for INCLUDE option in index creation + + * Adds option to allow simple DML commands from hot standby + + * Adds support for partitioned tables with replication factor > 1 + + * Prevents a deadlock on concurrent DROP TABLE and SELECT on Citus MX + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes a bug in MX mode, calling DROP SCHEMA with existing partitioned table + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + * Fixes a bug preventing rollback in CREATE PROCEDURE + + * Fixes a bug on not being able to drop index on a partitioned table + + * Fixes a bug on TRUNCATE when there is a foreign key to a reference table + + * Fixes a performance issue in prepared INSERT..SELECT + + * Fixes a bug which causes errors on DROP DATABASE IF EXISTS + + * Fixes a bug to remove intermediate result directory in pull-push execution + + * Improves query pushdown planning performance + + * Evaluate functions anywhere in query + + -- Burak Velioglu Fri, 02 Nov 2018 08:06:42 +0000 + +citus (7.5.1.citus-1) stable; urgency=low + + * Improves query pushdown planning performance + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + -- Burak Velioglu Wed, 29 Aug 2018 08:06:42 +0000 + +citus (7.4.2.citus-1) stable; urgency=low + + * Fixes a segfault in real-time executor during online shard move + + -- Mehmet Furkan Sahin Fri, 27 Jul 2018 13:42:27 +0000 + +citus (7.5.0.citus-1) stable; urgency=low + + * Adds foreign key support from hash distributed to reference tables + + * Adds SELECT ... FOR UPDATE support for router plannable queries + + * Adds support for non-partition columns in count distinct + + * Fixes a segfault in real-time executor during online shard move + + * Fixes ALTER TABLE ADD COLUMN constraint check + + * Fixes a bug where INSERT ... SELECT allows one to update dist. column + + * Allows DDL commands to be sequentialized via citus.multi_shard_modify_mode + + * Adds support for topn_union_agg and topn_add_agg across shards + + * Adds support for hll_union_agg and hll_add_agg across shards + + * Fixes a bug that might cause shards to have a wrong owner + + * Adds select_opens_transaction_block GUC + + * Adds utils to implement DDLs for policies in future + + * Makes intermediate results to use separate connections + + * Adds a node_conninfo GUC to set outgoing connection settings + + -- Mehmet Furkan Sahin Wed, 25 Jul 2018 9:32:24 +0000 + +citus (6.2.6.citus-1) stable; urgency=low + + * Adds support for respecting enable_hashagg in the master planner + + -- Burak Velioglu Fri, 06 Jul 2018 13:30:08 +0000 + +citus (7.4.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause txns to incorrectly proceed after failure + + * Fixes a bug on INSERT ... SELECT queries in prepared statements + + -- Burak Velioglu Wed, 20 Jun 2018 12:25:30 +0000 + +citus (7.2.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause SELECTs to crash during a rebalance + + -- Burak Velioglu Thu, 17 May 2018 11:51:56 +0000 + +citus (7.4.0.citus-1) stable; urgency=low + + * Adds support for non-pushdownable subqueries and CTEs in UPDATE/DELETE + + * Adds support for pushdownable subqueries and joins in UPDATE/DELETE + + * Adds faster shard pruning for subqueries + + * Adds partitioning support to MX table + + * Adds support for (VACUUM | ANALYZE) VERBOSE + + * Adds support for multiple ANDs in HAVING for pushdown planner + + * Adds support for quotation needy schema names + + * Improves operator check time in physical planner for custom data types + + * Removes broadcast join logic + + * Deprecates large_table_shard_count and master_expire_table_cache() + + * Modifies master_update_node to write-lock shards hosted by node over update + + * DROP TABLE now drops shards as the currrent user instead of the superuser + + * Adds specialised error codes for connection failures + + * Improves error messages on connection failure + + * Fixes issue which prevented multiple citus_table_size calls per query + + * Tests are updated to use create_distributed_table + + -- Burak Velioglu Tue, 15 May 2018 13:01:17 +0000 + +citus (7.3.0.citus-1) stable; urgency=low + + * Adds support for non-colocated joins between subqueries + + * Adds support for window functions that can be pushed down to worker + + * Adds support for modifying CTEs + + * Adds recursive plan for WHERE clause subqueries with recurring FROM clause + + * Adds support for bool_ and bit_ aggregates + + * Adds support for Postgres jsonb and json aggregation functions + + * Adds support for respecting enable_hashagg in the master plan + + * Performance improvements to reduce distributed planning time + + * Fixes a bug on planner when aggregate is used in ORDER BY + + * Fixes a bug on planner when DISTINCT (ON) clause is used with GROUP BY + + * Fixes a planner bug with distinct and aggregate clauses + + * Fixes a bug that opened new connections on each table size function call + + * Fixes a bug canceling backends not involved in distributed deadlocks + + * Fixes count distinct bug on column expressions when used with subqueries + + * Improves error handling on worker node failures + + * Improves error messages for INSERT queries that have subqueries + + -- Burak Velioglu Thu, 15 Mar 2018 14:16:10 +0000 + +citus (7.2.1.citus-1) stable; urgency=low + + * Fixes count distinct bug on column expressions when used with subqueries + + * Adds support for respecting enable_hashagg in the master plan + + * Fixes a bug canceling backends not involved in distributed deadlocks + + -- Burak Velioglu Tue, 06 Feb 2018 14:46:07 +0000 + +citus (7.2.0.citus-1) stable; urgency=low + + * Adds support for CTEs + + * Adds support for subqueries that require merge step + + * Adds support for set operations (UNION, INTERSECT, ...) + + * Adds support for 2PC auto-recovery + + * Adds support for querying local tables in CTEs and subqueries + + * Adds support for more SQL coverage in subqueries for reference tables + + * Adds support for count(distinct) in queries with a subquery + + * Adds support for non-equijoins when there is already an equijoin + + * Adds support for real-time executor to run in transaction blocks + + * Adds infrastructure for storing intermediate distributed query results + + * Adds a new GUC named enable_repartition_joins for auto executor switch + + * Adds support for limiting the intermediate result size + + * Improves support for queries with unions containing filters + + * Improves support for queries with unions containing joins + + * Improves support for subqueries in the WHERE clause + + * Increases COPY throughput + + * Enables pushing down queries containing only recurring tuples and GROUP BY + + * Load-balance queries that read from 0 shards + + * Improves support for using functions in subqueries + + * Fixes a bug that causing real-time executor to crash during cancellation + + * Fixes a bug that causing real-time executor to get stuck on cancellation + + * Fixes a bug that could block modification queries unnecessarily + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Fixes a bug that could cause an assert failure with ANALYZE statements + + * Fixes a bug that would push down wrong set operations in subqueries + + * Fixes a bug that could cause a deadlock in create_distributed_table + + * Fixes a bug that could confuse user about ANALYZE usage + + * Fixes a bug causing false positive distributed deadlock detections + + * Relaxes the locking for DDL commands on partitioned tables + + * Relaxes the locking on COPY with replication + + * Logs more remote commands when citus.log_remote_commands is set + + -- Burak Velioglu Tue, 16 Jan 2018 14:34:20 +0000 + +citus (6.2.5.citus-1) stable; urgency=low + + * Fixes a bug that could crash the coordinator while reporting a remote error + + -- Burak Velioglu Thu, 11 Jan 2018 11:40:28 +0000 + +citus (7.1.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Increases COPY throughput + + -- Burak Velioglu Fri, 05 Jan 2018 09:00:07 +0000 + +citus (7.1.1.citus-1) stable; urgency=low + + * Fixes a bug preventing pushing down subqueries with reference tables + + * Fixes a bug that could create false positive distributed deadlocks + + * Fixes a bug that could prevent running concurrent COPY and multi-shard DDL + + * Fixes a bug that could mislead users about ANALYZE queries + + -- Burak Velioglu Tue, 05 Dec 2017 09:00:07 +0000 + +citus (7.1.0.citus-1) stable; urgency=low + + * Adds support for native queries with multi shard UPDATE/DELETE queries + + * Expands reference table support in subquery pushdown + + * Adds window function support for subqueries and INSERT ... SELECT queries + + * Adds support for COUNT(DISTINCT) [ON] queries on non-partition columns + + * Adds support for DISTINCT [ON] queries on non-partition columns + + * Introduces basic usage statistic collector + + * Adds support for setting replica identity while creating distributed tables + + * Adds support for ALTER TABLE ... REPLICA IDENTITY queries + + * Adds pushdown support for LIMIT and HAVING grouped by partition key + + * Adds support for INSERT ... SELECT queries via worker nodes on MX clusters + + * Adds support for adding primary key using already defined index + + * Adds replication parameter to shard copy functions + + * Changes shard_name UDF to omit public schema name + + * Adds master_move_node UDF to make changes on nodename/nodeport more easy + + * Fixes a bug that could cause casting error with INSERT ... SELECT queries + + * Fixes a bug that could prevent upgrading servers from Citus 6.1 + + * Fixes a bug that could prevent attaching partitions to a table in schema + + * Fixes a bug preventing adding nodes to clusters with reference tables + + * Fixes a bug that could cause a crash with INSERT ... SELECT queries + + * Fixes a bug that could prevent creating a partitoned table on Cloud + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Velioglu Wed, 15 Nov 2017 09:00:07 +0000 + +citus (7.0.3.citus-1) stable; urgency=low + + * Fixes several bugs that could cause crash + + * Fixes a bug that could cause deadlock while creating reference tables + + * Fixes a bug that could cause false-positives in deadlock detection + + * Fixes a bug that could cause 2PC recovery not to work from MX workers + + * Fixes a bug that could cause cache incohorency + + * Fixes a bug that could cause maintenance daemon to skip cache invalidations + + * Improves performance of transaction recovery by using correct index + + -- Burak Yucesoy Mon, 16 Oct 2017 11:52:07 +0000 + +citus (7.0.2.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 22:29:01 +0000 + +citus (6.2.4.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 21:31:35 +0000 + +citus (6.1.3.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 20:31:35 +0000 + +citus (7.0.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause memory leaks in INSERT ... SELECT queries + + * Fixes a bug that could cause incorrect execution of prepared statements + + * Fixes a bug that could cause excessive memory usage during COPY + + * Incorporates latest changes from core PostgreSQL code + + -- Burak Yucesoy Tue, 12 Sep 2017 17:53:50 +0000 + +citus (7.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 10 + + * Drops support for PostgreSQL 9.5 + + * Adds support for multi-row INSERT + + * Adds support for router UPDATE and DELETE queries with subqueries + + * Adds infrastructure for distributed deadlock detection + + * Deprecates enable_deadlock_prevention flag + + * Adds support for partitioned tables + + * Adds support for creating UNLOGGED tables + + * Adds support for SAVEPOINT + + * Adds UDF citus_create_restore_point for taking distributed snapshots + + * Adds support for evaluating non-pushable INSERT ... SELECT queries + + * Adds support for subquery pushdown on reference tables + + * Adds shard pruning support for IN and ANY + + * Adds support for UPDATE and DELETE commands that prune down to 0 shard + + * Enhances transaction support by relaxing some transaction restrictions + + * Fixes a bug causing crash if distributed table has no shards + + * Fixes a bug causing crash when removing inactive node + + * Fixes a bug causing failure during COPY on tables with dropped columns + + * Fixes a bug causing failure during DROP EXTENSION + + * Fixes a bug preventing executing VACUUM and INSERT concurrently + + * Fixes a bug in prepared INSERT statements containing an implicit cast + + * Fixes several issues related to statement cancellations and connections + + * Fixes several 2PC related issues + + * Removes an unnecessary dependency causing warning messages in pg_dump + + * Adds internal infrastructure for follower clusters + + * Adds internal infrastructure for progress tracking + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Mon, 28 Aug 2017 12:27:53 +0000 + +citus (6.2.3.citus-1) stable; urgency=low + + * Fixes a crash during execution of local CREATE INDEX CONCURRENTLY + + * Fixes a bug preventing usage of quoted column names in COPY + + * Fixes a bug in prepared INSERTs with implicit cast in partition column + + * Relaxes locks in VACUUM to ensure concurrent execution with INSERT + + -- Burak Yucesoy Thu, 13 Jul 2017 11:27:17 +0000 + +citus (6.2.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Burak Velioglu Wed, 07 Jun 2017 09:42:17 +0000 + +citus (6.1.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Jason Petersen Wed, 31 May 2017 16:14:11 +0000 + +citus (6.2.1.citus-1) stable; urgency=low + + * Relaxes version-check logic to avoid breaking non-distributed commands + + -- Jason Petersen Wed, 24 May 2017 22:36:07 +0000 + +citus (6.2.0.citus-1) stable; urgency=low + + * Increases SQL subquery coverage by pushing down more kinds of queries + + * Adds CustomScan API support to allow read-only transactions + + * Adds support for CREATE/DROP INDEX CONCURRENTLY + + * Adds support for ALTER TABLE ... ADD CONSTRAINT + + * Adds support for ALTER TABLE ... RENAME COLUMN + + * Adds support for DISABLE/ENABLE TRIGGER ALL + + * Adds support for expressions in the partition column in INSERTs + + * Adds support for query parameters in combination with function evaluation + + * Adds support for creating distributed tables from non-empty local tables + + * Adds UDFs to get size of distributed tables + + * Adds UDFs to add a new node without replicating reference tables + + * Adds checks to prevent running Citus binaries with wrong metadata tables + + * Improves shard pruning performance for range queries + + * Improves planner performance for joins involving co-located tables + + * Improves shard copy performance by creating indexes after copy + + * Improves task-tracker performance by batching several status checks + + * Enables router planner for queries on range partitioned table + + * Changes TRUNCATE to drop local data only if enable_ddl_propagation is off + + * Starts to execute DDL on coordinator before workers + + * Fixes a bug causing incorrectly reading invalidated cache + + * Fixes a bug related to creation of schemas in workers with incorrect owner + + * Fixes a bug related to concurrent run of shard drop functions + + * Fixes a bug related to EXPLAIN ANALYZE with DML queries + + * Fixes a bug related to SQL functions in FROM clause + + * Adds a GUC variable to report cross shard queries + + * Fixes a bug related to partition columns without native hash function + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Tue, 16 May 2017 16:05:22 +0000 + +citus (6.1.1.citus-1) stable; urgency=low + + * Fixes a crash caused by router executor use after connection timeouts + + * Fixes a crash caused by relation cache invalidation during COPY + + * Fixes bug related to DDL use within PL/pgSQL functions + + * Fixes a COPY bug related to types lacking binary output functions + + * Fixes a bug related to modifications with parameterized partition values + + * Fixes improper value interpolation in worker sequence generation + + * Guards shard pruning logic against zero-shard tables + + * Fixes possible NULL pointer dereference and buffer underflow, via PVS-Studio + + * Fixes a INSERT...SELECT bug that could push down non-partition column JOINs + + -- Metin Doslu Fri, 5 May 2017 17:42:00 +0000 + +citus (6.1.0.citus-1) stable; urgency=low + + * Implements reference tables, transactionally replicated to all nodes + + * Adds upgrade_to_reference_table UDF to upgrade pre-6.1 reference tables + + * Expands prepared statement support to nearly all statements + + * Adds support for creating VIEWs which reference distributed tables + + * Adds targeted VACUUM/ANALYZE support + + * Adds support for the FILTER clause in aggregate expressions + + * Adds support for function evaluation within INSERT INTO ... SELECT + + * Adds support for creating foreign key constraints with ALTER TABLE + + * Adds logic to choose router planner for all queries it supports + + * Enhances create_distributed_table with parameter for explicit colocation + + * Adds generally useful utility UDFs previously available as "Citus Tools" + + * Adds user-facing UDFs for locking shard resources and metadata + + * Refactors connection and transaction management for more consistency + + * Enhances COPY with fully transactional semantics + + * Improves support for cancellation for a number of queries and commands + + * Adds column_to_column_name UDF to help users understand partkey values + + * Adds master_disable_node UDF for temporarily disabling nodes + + * Adds proper MX ("masterless") metadata propagation logic + + * Adds start_metadata_sync_to_node UDF to propagate metadata changes to nodes + + * Enhances SERIAL compatibility with MX tables + + * Adds an node_connection_timeout parameter to set node connection timeouts + + * Adds enable_deadlock_prevention setting to permit multi-node transactions + + * Adds a replication_model setting to specify replication of new tables + + * Changes the shard_replication_factor setting's default value to one + + * Adds code to automatically set max_prepared_transactions if not configured + + * Accelerates lookup of colocated shard placements + + * Fixes a bug affecting INSERT INTO ... SELECT queries using constant values + + * Fixes a bug by ensuring COPY does not mark placements inactive + + * Fixes a bug affecting reads from pg_dist_shard_placement table + + * Fixes a crash triggered by creating a foreign key without a column + + * Fixes a crash related to accessing catalog tables after aborted transaction + + * Fixes a bug affecting JOIN queries requiring repartitions + + * Fixes a bug affecting node insertions to pg_dist_node table + + * Fixes a crash triggered by queries with modifying common table expressions + + * Fixes a bug affecting workloads with concurrent shard appends and deletions + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Thu, 9 Feb 2017 16:17:41 +0000 + +citus (6.0.1.citus-3) stable; urgency=low + + * First build using new versioning practices + + -- Jason Petersen Wed, 8 Feb 2017 23:19:46 +0000 + +citus (6.0.1.citus-2) stable; urgency=low + + * Transitional package to guide users to new package name + + -- Jason Petersen Mon, 6 Feb 2017 16:33:44 +0000 + +citus (6.0.1.citus-1) stable; urgency=low + + * Fixes a bug causing failures during pg_upgrade + + * Fixes a bug preventing DML queries during colocated table creation + + * Fixes a bug that caused NULL parameters to be incorrectly passed as text + + -- Burak Yucesoy Wed, 30 Nov 2016 15:27:38 +0000 + +citus (6.0.0.citus-1) stable; urgency=low + + * Adds compatibility with PostgreSQL 9.6, now the recommended version + + * Removes the pg_worker_list.conf file in favor of a pg_dist_node table + + * Adds master_add_node and master_add_node UDFs to manage membership + + * Removes the \stage command and corresponding csql binary in favor of COPY + + * Removes copy_to_distributed_table in favor of first-class COPY support + + * Adds support for multiple DDL statements within a transaction + + * Adds support for certain foreign key constraints + + * Adds support for parallel INSERT INTO ... SELECT against colocated tables + + * Adds support for the TRUNCATE command + + * Adds support for HAVING clauses in SELECT queries + + * Adds support for EXCLUDE constraints which include the partition column + + * Adds support for system columns in queries (tableoid, ctid, etc.) + + * Adds support for relation name extension within INDEX definitions + + * Adds support for no-op UPDATEs of the partition column + + * Adds several general-purpose utility UDFs to aid in Citus maintenance + + * Adds master_expire_table_cache UDF to forcibly expire cached shards + + * Parallelizes the processing of DDL commands which affect distributed tables + + * Adds support for repartition jobs using composite or custom types + + * Enhances object name extension to handle long names and large shard counts + + * Parallelizes the master_modify_multiple_shards UDF + + * Changes distributed table creation to error if target table is not empty + + * Changes the pg_dist_shard.logicalrelid column from an oid to regclass + + * Adds a placementid column to pg_dist_shard_placement, replacing Oid use + + * Removes the pg_dist_shard.shardalias distribution metadata column + + * Adds pg_dist_partition.repmodel to track tables using streaming replication + + * Adds internal infrastructure to take snapshots of distribution metadata + + * Addresses the need to invalidate prepared statements on metadata changes + + * Adds a mark_tables_colocated UDF for denoting pre-6.0 manual colocation + + * Fixes a bug affecting prepared statement execution within PL/pgSQL + + * Fixes a bug affecting COPY commands using composite types + + * Fixes a bug that could cause crashes during EXPLAIN EXECUTE + + * Separates worker and master job temporary folders + + * Eliminates race condition between distributed modification and repair + + * Relaxes the requirement that shard repairs also repair colocated shards + + * Implements internal functions to track which tables' shards are colocated + + * Adds pg_dist_partition.colocationid to track colocation group membership + + * Extends shard copy and move operations to respect colocation settings + + * Adds pg_dist_local_group to prepare for future MX-related changes + + * Adds create_distributed_table to easily create shards and infer colocation + + -- Jason Petersen Tue, 8 Nov 2016 19:45:45 +0000 + +citus (5.2.2.citus-1) stable; urgency=low + + * Adds support for IF NOT EXISTS clause of CREATE INDEX command + + * Adds support for RETURN QUERY and FOR ... IN PL/pgSQL features + + * Extends the router planner to handle more queries + + * Changes COUNT of zero-row sets to return 0 rather than an empty result + + * Reduces the minimum permitted task_tracker_delay to a single millisecond + + * Fixes a bug that caused crashes during joins with a WHERE false clause + + * Fixes a bug triggered by unique violation errors raised in long txns + + * Fixes a bug resulting in multiple registration of transaction callbacks + + * Fixes a bug which could result in stale reads of distribution metadata + + * Fixes a bug preventing distributed modifications in some PL/pgSQL functions + + * Fixes some code paths that could hypothetically read uninitialized memory + + * Lowers log level of "waiting for activity" messages + + -- Jason Petersen Tue, 8 Nov 2016 18:43:37 +0000 + +citus (5.2.1.citus-1) stable; urgency=low + + * Fixes subquery pushdown to properly extract outer join qualifiers + + * Addresses possible memory leak during multi-shard transactions + + -- Jason Petersen Tue, 6 Sep 2016 20:47:15 +0000 + +citus (5.2.0.citus-1) stable; urgency=low + + * Drops support for PostgreSQL 9.4; PostgreSQL 9.5 is required + + * Adds schema support for tables, named objects (types, operators, etc.) + + * Evaluates non-immutable functions on master in all modification commands + + * Adds support for SERIAL types in non-partition columns + + * Adds support for RETURNING clause in INSERT, UPDATE, and DELETE commands + + * Adds support for multi-statement transactions using a fixed set of nodes + + * Full SQL support for SELECT queries which can be executed on single worker + + * Adds option to perform DDL changes using prepared transactions (2PC) + + * Adds an enable_ddl_propagation parameter to control DDL propagation + + * Accelerates shard pruning during merges + + * Adds master_modify_multiple_shards UDF to modify many shards at once + + * Adds COPY support for arrays of user-defined types + + * Now supports parameterized prepared statements for certain use cases + + * Extends LIMIT/OFFSET support to all executor types + + * Constraint violations now fail fast rather than hitting all placements + + * Makes master_create_empty_shard aware of shard placement policy + + * Reduces unnecessary sleep during queries processed by real-time executor + + * Improves task tracker executor's task cleanup logic + + * Relaxes restrictions on cancellation of DDL commands + + * Removes ONLY keyword from worker SELECT queries + + * Error message improvements and standardization + + * Moves master_update_shard_statistics function to pg_catalog schema + + * Fixes a bug where hash-partitioned anti-joins could return bad results + + * Now sets storage type correctly for foreign table-backed shards + + * Fixes master_update_shard_statistics issue with hash-partitioned tables + + * Fixes an issue related to extending table names that require escaping + + * Reduces risk of row counter overflows during modifications + + * Fixes a crash related to FILTER clause use in COUNT DISTINCT subqueries + + * Fixes crashes related to partition columns with high attribute numbers + + * Fixes certain subquery and join crashes + + * Detects flex for build even if PostgreSQL was built without it + + * Fixes assert-enabled crash when all_modifications_commutative is true + + -- Jason Petersen Wed, 17 Aug 2016 10:23:21 +0000 + +citus (5.2.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.2. + + -- Jason Petersen Mon, 01 Aug 2016 17:01:05 +0000 + +citus (5.1.1-1) stable; urgency=low + + * Adds complex count distinct expression support in repartitioned subqueries + + * Improves task tracker job cleanup logic, addressing a memory leak + + * Fixes bug that generated incorrect results for LEFT JOIN queries + + * Improves compatibility with Debian's reproducible builds project + + * Fixes build issues on FreeBSD platforms + + -- Jason Petersen Fri, 17 Jun 2016 16:20:15 +0000 + +citus (5.1.0-1) stable; urgency=low + + * Adds distributed COPY to rapidly populate distributed tables + + * Adds support for using EXPLAIN on distributed queries + + * Recognizes and fast-paths single-shard SELECT statements automatically + + * Increases INSERT throughput via shard pruning optimizations + + * Improves planner performance for joins involving tables with many shards + + * Adds ability to pass columns as arguments to function calls in UPDATEs + + * Introduces transaction manager for use by multi-shard commands + + * Adds COUNT(DISTINCT ...) pushdown optimization for hash-partitioned tables + + * Adds support for some UNIQUE indexes on hash- or range-partitioned tables + + * Deprecates \stage in favor of using COPY for append-partition tables + + * Deprecates copy_to_distributed_table in favor of first-class COPY support + + * Fixes build problems when using non-packaged PostgreSQL installs + + * Fixes bug that sometimes skipped pruning when partitioned by VARCHAR column + + * Fixes bug impeding use of user functions in repartitioned subqueries + + * Fixes bug involving queries with equality comparisons of boolean types + + * Fixes crash that prevented use alongside pg_stat_statements + + * Fixes crash arising from SELECT queries that lack a target list + + * Improves warning and error messages + + -- Jason Petersen Tue, 17 May 2016 16:55:02 +0000 + +citus (5.1.0~rc.2-1) testing; urgency=low + + * Fix EXPLAIN output when FORMAT JSON in use + + -- Jason Petersen Mon, 16 May 2016 11:08:09 +0000 + +citus (5.1.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.1. + + -- Jason Petersen Wed, 04 May 2016 19:26:23 +0000 + +citus (5.0.1-1) stable; urgency=low + + * Fixes issues on 32-bit systems + + -- Jason Petersen Fri, 15 Apr 2016 19:17:35 +0000 + +citus (5.0.0-1) stable; urgency=low + + * Initial release + + -- Jason Petersen Thu, 24 Mar 2016 10:12:52 -0400 diff --git a/python/pkgvars b/python/pkgvars new file mode 100644 index 00000000..2ad71e5c --- /dev/null +++ b/python/pkgvars @@ -0,0 +1,6 @@ +pkgname=citus +pkgdesc='Citus (Open-Source)' +pkglatest=10.0.3-1 +releasepg=11,12,13 +nightlypg=12,13 +versioning=fancy \ No newline at end of file diff --git a/python/publish-packages.py b/python/publish-packages.py new file mode 100644 index 00000000..e09fc9bf --- /dev/null +++ b/python/publish-packages.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +from subprocess import PIPE +import subprocess +import os +import json +import re +import time +import sys +from pprint import pprint + +ms_package_repo_map = { + "el/8": "centos-8", "el/7": "centos-7", "debian/buster": "debian-buster", "debian/jessie": "debian-jessie", + "debian/stretch": "debian-stretch", "ubuntu/bionic": "ubuntu-bionic", "ubuntu/xenial": "ubuntu-xenial", + "ubuntu/focal": "ubuntu-focal"} + + +def run(command, *args, **kwargs): + print(command) + result = subprocess.run(command, *args, check=True, shell=True, **kwargs) + print() + return result + + +result = run("repoclient repo list", stdout=PIPE) + +all_repos = json.loads(result.stdout) + +citus_repos = {} +for repo in all_repos: + if not repo["url"].startswith("citus-"): + continue + name = repo["url"][len("citus-"):] + if name in ("ubuntu", "debian"): + # Suffix distribution + name = name + "-" + repo["distribution"] + else: + # Put dash before numberrepo + name = re.sub(r"(\d+)", r"-\1", name) + citus_repos[name] = repo + +# pprint(citus_repos) +target_platform = sys.argv[1] +submission_responses = {} +print("Citus Repos") +print("Current Dir" + os.getcwd()) +pprint(citus_repos) +for package_file in os.listdir("pkgs/releases"): + + print("Target Platform is " + target_platform) + repo_platform = ms_package_repo_map[target_platform] + repo = citus_repos[repo_platform] + package_path = os.path.join("pkgs/releases", package_file) + + print("Repo Url:" + repo["url"]) + # Ensure deb packages contain the distribution, so they do not conflict + if repo["url"] in ("citus-ubuntu", "citus-debian"): + if repo["distribution"] not in package_file: + if not package_path.endswith("amd64.deb"): + raise "Package should have ended with amd64.deb: %s" % package_path + old_package_path = package_path + package_prefix = package_path[: -len("amd64.deb")] + package_path = "%s+%s_amd64.deb" % ( + package_prefix, + repo["distribution"], + ) + os.rename(old_package_path, package_path) + + # Publish packages + if os.path.isfile(package_path) and (package_file.endswith(".rpm") or package_file.endswith(".deb")): + result = run( + "repoclient package add --repoID %s %s" % (repo["id"], package_path), + stdout=PIPE, + ) + submission_responses[package_path] = json.loads(result.stdout) + print( + "Waiting for 30 seconds to avoid concurrency problems on publishing server" + ) + time.sleep(30) + +pprint(submission_responses) + +# Check if published successfully +for i in range(15): + unfinished_submissions = {} + for package_path, response in submission_responses.items(): + package_id = response["Location"].split("/")[-1] + + try: + run("repoclient package check %s" % package_id) + except Exception: + print(package_path, "was not finished publishing") + unfinished_submissions[package_path] = response + + if not unfinished_submissions: + break + submission_responses = unfinished_submissions + time.sleep(i) + +if unfinished_submissions: + print("The following packages were not published successfuly") + pprint(unfinished_submissions) + raise Exception("Some packages were not finished publishing") diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 00000000..46304c9e --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,73 @@ +adal==1.2.2 +appdirs==1.4.3 +attrs==19.3.0 +Automat==0.8.0 +blinker==1.4 +certifi==2019.11.28 +chardet==3.0.4 +Click==7.0 +cloud-init==20.4 +colorama==0.4.3 +command-not-found==0.3 +configobj==5.0.6 +constantly==15.1.0 +cryptography==2.8 +dbus-python==1.2.16 +distlib==0.3.0 +distro==1.4.0 +distro-info===0.23ubuntu1 +entrypoints==0.3 +filelock==3.0.12 +httplib2==0.14.0 +hyperlink==19.0.0 +idna==2.8 +importlib-metadata==1.5.0 +incremental==16.10.1 +Jinja2==2.10.1 +jsonpatch==1.22 +jsonpointer==2.0 +jsonschema==3.2.0 +keyring==18.0.1 +language-selector==0.1 +launchpadlib==1.10.13 +lazr.restfulclient==0.14.2 +lazr.uri==1.0.3 +MarkupSafe==1.1.0 +more-itertools==4.2.0 +netifaces==0.10.4 +oauthlib==3.1.0 +pathlib2==2.3.5 +pexpect==4.6.0 +pyasn1==0.4.2 +pyasn1-modules==0.2.1 +Pygments==2.3.1 +PyGObject==3.36.0 +PyHamcrest==1.9.0 +PyJWT==1.7.1 +pymacaroons==0.13.0 +PyNaCl==1.3.0 +pyOpenSSL==19.0.0 +pyrsistent==0.15.5 +pyserial==3.4 +python-apt==2.0.0+ubuntu0.20.4.4 +python-dateutil==2.7.3 +python-debian===0.1.36ubuntu1 +PyYAML==5.3.1 +requests==2.22.0 +requests-unixsocket==0.2.0 +SecretStorage==2.3.1 +service-identity==18.1.0 +simplejson==3.16.0 +six==1.14.0 +sos==4.0 +ssh-import-id==5.10 +systemd-python==234 +Twisted==18.9.0 +ubuntu-advantage-tools==20.3 +ufw==0.36 +unattended-upgrades==0.1 +urllib3==1.25.8 +virtualenv==20.0.17 +wadllib==1.3.3 +zipp==1.0.0 +zope.interface==4.7.1 diff --git a/python/templates/pkgvars.tmpl b/python/templates/pkgvars.tmpl new file mode 100644 index 00000000..7e41782d --- /dev/null +++ b/python/templates/pkgvars.tmpl @@ -0,0 +1,6 @@ +pkgname=citus +pkgdesc='Citus (Open-Source)' +pkglatest={{version}} +releasepg=11,12,13 +nightlypg=12,13 +versioning=fancy \ No newline at end of file diff --git a/python/templates/project.spec.tmpl b/python/templates/project.spec.tmpl new file mode 100644 index 00000000..2edbe668 --- /dev/null +++ b/python/templates/project.spec.tmpl @@ -0,0 +1,82 @@ +%global pgmajorversion 11 +%global pgpackageversion 11 +%global pginstdir /usr/pgsql-%{pgpackageversion} +%global sname citus + +Summary: PostgreSQL-based distributed RDBMS +Name: %{sname}%{?pkginfix}_%{pgmajorversion} +Provides: %{sname}_%{pgmajorversion} +Conflicts: %{sname}_%{pgmajorversion} +Version: {{fancy_version_str}} +Release: {{fancy_version_no}}%{dist} +License: AGPLv3 +Group: Applications/Databases +Source0: https://github.com/citusdata/{{project_name}}/archive/v10.0.3.tar.gz +URL: https://github.com/citusdata/{{project_name}} +BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel +Requires: postgresql%{pgmajorversion}-server +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +%description +Citus horizontally scales PostgreSQL across commodity servers +using sharding and replication. Its query engine parallelizes +incoming SQL queries across these servers to enable real-time +responses on large datasets. + +Citus extends the underlying database rather than forking it, +which gives developers and enterprises the power and familiarity +of a traditional relational database. As an extension, Citus +supports new PostgreSQL releases, allowing users to benefit from +new features while maintaining compatibility with existing +PostgreSQL tools. Note that Citus supports many (but not all) SQL +commands. + +%prep +%setup -q -n %{sname}-%{version} + +%build +%configure PG_CONFIG=%{pginstdir}/bin/pg_config --with-extra-version="%{?conf_extra_version}" --with-security-flags +make %{?_smp_mflags} + +%install +%make_install +# Install documentation with a better name: +%{__mkdir} -p %{buildroot}%{pginstdir}/doc/extension +%{__cp} README.md %{buildroot}%{pginstdir}/doc/extension/README-%{sname}.md +# Set paths to be packaged other than LICENSE, README & CHANGELOG.md +echo %{pginstdir}/include/server/citus_*.h >> installation_files.list +echo %{pginstdir}/include/server/distributed/*.h >> installation_files.list +echo %{pginstdir}/lib/%{sname}.so >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}-*.sql >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list +%ifarch ppc64 ppc64le + %else + %if 0%{?rhel} && 0%{?rhel} <= 6 + %else + echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list + + # Columnar does not exist in Citus versions < 10.0 + # At this point, we don't have %{pginstdir}, + # so first check build directory for columnar. + [[ -d %{buildroot}%{pginstdir}/lib/bitcode/columnar/ ]] && echo %{pginstdir}/lib/bitcode/columnar/*.bc >> installation_files.list + %endif +%endif + +%clean +%{__rm} -rf %{buildroot} + +%files -f installation_files.list +%files +%defattr(-,root,root,-) +%doc CHANGELOG.md +%if 0%{?rhel} && 0%{?rhel} <= 6 +%doc LICENSE +%else +%license LICENSE +%endif +%doc %{pginstdir}/doc/extension/README-%{sname}.md + +%changelog +{{changelog}} \ No newline at end of file diff --git a/python/templates/showpersons.txt b/python/templates/showpersons.txt new file mode 100644 index 00000000..dc32f91c --- /dev/null +++ b/python/templates/showpersons.txt @@ -0,0 +1,3 @@ +{% for person in persons -%} + {{ person.name }} {{ person.age }} +{% endfor %} \ No newline at end of file diff --git a/python/tests/files/citus.spec b/python/tests/files/citus.spec new file mode 100644 index 00000000..6bbd61be --- /dev/null +++ b/python/tests/files/citus.spec @@ -0,0 +1,318 @@ +%global pgmajorversion 11 +%global pgpackageversion 11 +%global pginstdir /usr/pgsql-%{pgpackageversion} +%global sname citus + +Summary: PostgreSQL-based distributed RDBMS +Name: %{sname}%{?pkginfix}_%{pgmajorversion} +Provides: %{sname}_%{pgmajorversion} +Conflicts: %{sname}_%{pgmajorversion} +Version: 10.0.3-1 +Release: 1%{dist} +License: AGPLv3 +Group: Applications/Databases +Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz +URL: https://github.com/citusdata/ +BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel +Requires: postgresql%{pgmajorversion}-server +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +%description +Citus horizontally scales PostgreSQL across commodity servers +using sharding and replication. Its query engine parallelizes +incoming SQL queries across these servers to enable real-time +responses on large datasets. + +Citus extends the underlying database rather than forking it, +which gives developers and enterprises the power and familiarity +of a traditional relational database. As an extension, Citus +supports new PostgreSQL releases, allowing users to benefit from +new features while maintaining compatibility with existing +PostgreSQL tools. Note that Citus supports many (but not all) SQL +commands. + +%prep +%setup -q -n %{sname}-%{version} + +%build +%configure PG_CONFIG=%{pginstdir}/bin/pg_config --with-extra-version="%{?conf_extra_version}" --with-security-flags +make %{?_smp_mflags} + +%install +%make_install +# Install documentation with a better name: +%{__mkdir} -p %{buildroot}%{pginstdir}/doc/extension +%{__cp} README.md %{buildroot}%{pginstdir}/doc/extension/README-%{sname}.md +# Set paths to be packaged other than LICENSE, README & CHANGELOG.md +echo %{pginstdir}/include/server/citus_*.h >> installation_files.list +echo %{pginstdir}/include/server/distributed/*.h >> installation_files.list +echo %{pginstdir}/lib/%{sname}.so >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}-*.sql >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list +%ifarch ppc64 ppc64le + %else + %if 0%{?rhel} && 0%{?rhel} <= 6 + %else + echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list + + # Columnar does not exist in Citus versions < 10.0 + # At this point, we don't have %{pginstdir}, + # so first check build directory for columnar. + [[ -d %{buildroot}%{pginstdir}/lib/bitcode/columnar/ ]] && echo %{pginstdir}/lib/bitcode/columnar/*.bc >> installation_files.list + %endif +%endif + +%clean +%{__rm} -rf %{buildroot} + +%files -f installation_files.list +%files +%defattr(-,root,root,-) +%doc CHANGELOG.md +%if 0%{?rhel} && 0%{?rhel} <= 6 +%doc LICENSE +%else +%license LICENSE +%endif +%doc %{pginstdir}/doc/extension/README-%{sname}.md + +%changelog +* Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 +- Update to Citus 10.0.3 + +* Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 +- Official 10.0.2 release of Citus + +* Wed Jan 27 2021 - gurkanindibay 9.5.2.citus-1 +- Official 9.5.2 release of Citus + +* Tue Jan 5 2021 - gurkanindibay 9.4.4.citus-1 +- Official 9.4.4 release of Citus + +* Wed Dec 2 2020 - Onur Tirtir 9.5.1.citus-1 +- Official 9.5.1 release of Citus + +* Tue Nov 24 2020 - Onur Tirtir 9.4.3.citus-1 +- Official 9.4.3 release of Citus + +* Wed Nov 11 2020 - Onur Tirtir 9.5.0.citus-1 +- Official 9.5.0 release of Citus + +* Thu Oct 22 2020 - Onur Tirtir 9.4.2.citus-1 +- Official 9.4.2 release of Citus + +* Wed Sep 30 2020 - Onur Tirtir 9.4.1.citus-1 +- Official 9.4.1 release of Citus + +* Tue Jul 28 2020 - Onur Tirtir 9.4.0.citus-1 +- Official 9.4.0 release of Citus + +* Mon Jul 27 2020 - Onur Tirtir 9.3.5.citus-1 +- Official 9.3.5 release of Citus + +* Wed Jul 22 2020 - Onur Tirtir 9.3.4.citus-1 +- Official 9.3.4 release of Citus + +* Mon Jul 13 2020 - Onur Tirtir 9.3.3.citus-1 +- Official 9.3.3 release of Citus + +* Thu May 7 2020 - Onur Tirtir 9.3.0.citus-1 +- Update to Citus 9.3.0 + +* Tue Mar 31 2020 - Onur Tirtir 9.2.4.citus-1 +- Update to Citus 9.2.4 + +* Thu Mar 26 2020 - Onur Tirtir 9.2.3.citus-1 +- Update to Citus 9.2.3 + +* Fri Mar 6 2020 - Onur Tirtir 9.0.2.citus-1 +- Update to Citus 9.0.2 + +* Fri Mar 6 2020 - Onur Tirtir 9.2.2.citus-1 +- Update to Citus 9.2.2 + +* Fri Feb 14 2020 - Onur Tirtir 9.2.1.citus-1 +- Update to Citus 9.2.1 + +* Mon Feb 10 2020 - Onur Tirtir 9.2.0.citus-1 +- Update to Citus 9.2.0 + +* Wed Dec 18 2019 - Onur Tirtir 9.1.1.citus-1 +- Update to Citus 9.1.1 + +* Thu Nov 28 2019 - Onur Tirtir 9.1.0.citus-1 +- Update to Citus 9.1.0 + +* Wed Oct 30 2019 - Hanefi Onaldi 9.0.1.citus-1 +- Update to Citus 9.0.1 + +* Thu Oct 10 2019 - Hanefi Onaldi 9.0.0.citus-1 +- Update to Citus 9.0.0 + +* Fri Aug 9 2019 - Hanefi Onaldi 8.3.2.citus-1 +- Update to Citus 8.3.2 + +* Mon Jul 29 2019 - Hanefi Onaldi 8.3.1.citus-1 +- Update to Citus 8.3.1 + +* Wed Jul 10 2019 - Burak Velioglu 8.3.0.citus-1 +- Update to Citus 8.3.0 + +* Wed Jun 12 2019 - Burak Velioglu 8.2.2.citus-1 +- Update to Citus 8.2.2 + +* Wed Apr 3 2019 - Burak Velioglu 8.2.1.citus-1 +- Update to Citus 8.2.1 + +* Wed Apr 3 2019 - Burak Velioglu 8.1.2.citus-1 +- Update to Citus 8.1.2 + +* Thu Mar 28 2019 - Burak Velioglu 8.2.0.citus-1 +- Update to Citus 8.2.0 + +* Wed Jan 9 2019 - Burak Velioglu 8.0.3.citus-1 +- Update to Citus 8.0.3 + +* Mon Jan 7 2019 - Burak Velioglu 8.1.1.citus-1 +- Update to Citus 8.1.1 + +* Tue Dec 18 2018 - Burak Velioglu 8.1.0.citus-1 +- Update to Citus 8.1.0 + +* Thu Dec 13 2018 - Burak Velioglu 8.0.2.citus-1 +- Update to Citus 8.0.2 + +* Wed Dec 12 2018 - Burak Velioglu 7.5.4.citus-1 +- Update to Citus 7.5.4 + +* Wed Nov 28 2018 - Burak Velioglu 8.0.1.citus-1 +- Update to Citus 8.0.1 + +* Wed Nov 28 2018 - Burak Velioglu 7.5.3.citus-1 +- Update to Citus 7.5.3 + +* Wed Nov 14 2018 - Burak Velioglu 7.5.2.citus-1 +- Update to Citus 7.5.2 + +* Fri Nov 02 2018 - Burak Velioglu 8.0.0.citus-1 +- Update to Citus 8.0.0 + +* Wed Aug 29 2018 - Burak Velioglu 7.5.1.citus-1 +- Update to Citus 7.5.1 + +* Fri Jul 27 2018 - Mehmet Furkan Sahin 7.4.2.citus-1 +- Update to Citus 7.4.2 + +* Wed Jul 25 2018 - Mehmet Furkan Sahin 7.5.0.citus-1 +- Update to Citus 7.5.0 + +* Wed Jun 20 2018 - Burak Velioglu 7.4.1.citus-1 +- Update to Citus 7.4.1 + +* Thu May 17 2018 - Burak Velioglu 7.2.2.citus-1 +- Update to Citus 7.2.2 + +* Tue May 15 2018 - Burak Velioglu 7.4.0.citus-1 +- Update to Citus 7.4.0 + +* Thu Mar 15 2018 - Burak Velioglu 7.3.0.citus-1 +- Update to Citus 7.3.0 + +* Tue Feb 6 2018 - Burak Velioglu 7.2.1.citus-1 +- Update to Citus 7.2.1 + +* Tue Jan 16 2018 - Burak Velioglu 7.2.0.citus-1 +- Update to Citus 7.2.0 + +* Thu Jan 11 2018 - Burak Velioglu 6.2.5.citus-1 +- Update to Citus 6.2.5 + +* Fri Jan 05 2018 - Burak Velioglu 7.1.2.citus-1 +- Update to Citus 7.1.2 + +* Tue Dec 05 2017 - Burak Velioglu 7.1.1.citus-1 +- Update to Citus 7.1.1 + +* Wed Nov 15 2017 - Burak Velioglu 7.1.0.citus-1 +- Update to Citus 7.1.0 + +* Mon Oct 16 2017 - Burak Yucesoy 7.0.3.citus-1 +- Update to Citus 7.0.3 + +* Thu Sep 28 2017 - Burak Yucesoy 7.0.2.citus-1 +- Update to Citus 7.0.2 + +* Thu Sep 28 2017 - Burak Yucesoy 6.2.4.citus-1 +- Update to Citus 6.2.4 + +* Thu Sep 28 2017 - Burak Yucesoy 6.1.3.citus-1 +- Update to Citus 6.1.3 + +* Tue Sep 12 2017 - Burak Yucesoy 7.0.1.citus-1 +- Update to Citus 7.0.1 + +* Tue Aug 29 2017 - Burak Yucesoy 7.0.0.citus-1 +- Update to Citus 7.0.0 + +* Thu Jul 13 2017 - Burak Yucesoy 6.2.3.citus-1 +- Update to Citus 6.2.3 + +* Wed Jun 7 2017 - Burak Velioglu 6.2.2.citus-1 +- Update to Citus 6.2.2 + +* Wed Jun 7 2017 - Jason Petersen 6.1.2.citus-1 +- Update to Citus 6.1.2 + +* Wed May 24 2017 - Jason Petersen 6.2.1.citus-1 +- Update to Citus 6.2.1 + +* Tue May 16 2017 - Burak Yucesoy 6.2.0.citus-1 +- Update to Citus 6.2.0 + +* Fri May 5 2017 - Metin Doslu 6.1.1.citus-1 +- Update to Citus 6.1.1 + +* Thu Feb 9 2017 - Burak Yucesoy 6.1.0.citus-1 +- Update to Citus 6.1.0 + +* Wed Feb 8 2017 - Jason Petersen 6.0.1.citus-2 +- Transitional package to guide users to new package name + +* Wed Nov 30 2016 - Burak Yucesoy 6.0.1.citus-1 +- Update to Citus 6.0.1 + +* Tue Nov 8 2016 - Jason Petersen 6.0.0.citus-1 +- Update to Citus 6.0.0 + +* Tue Nov 8 2016 - Jason Petersen 5.2.2.citus-1 +- Update to Citus 5.2.2 + +* Tue Sep 6 2016 - Jason Petersen 5.2.1.citus-1 +- Update to Citus 5.2.1 + +* Wed Aug 17 2016 - Jason Petersen 5.2.0.citus-1 +- Update to Citus 5.2.0 + +* Mon Aug 1 2016 - Jason Petersen 5.2.0-0.1.rc.1 +- Release candidate for 5.2 + +* Fri Jun 17 2016 - Jason Petersen 5.1.1-1 +- Update to Citus 5.1.1 + +* Tue May 17 2016 - Jason Petersen 5.1.0-1 +- Update to Citus 5.1.0 + +* Mon May 16 2016 - Jason Petersen 5.1.0-0.2.rc.2 +- Fix EXPLAIN output when FORMAT JSON in use + +* Wed May 4 2016 - Jason Petersen 5.1.0-0.1.rc.1 +- Release candidate for 5.1 + +* Fri Apr 15 2016 - Jason Petersen 5.0.1-1 +- Fixes issues on 32-bit systems + +* Fri Mar 25 2016 - Devrim Gündüz 5.0.0-1 +- Initial RPM packaging for PostgreSQL RPM Repository, + based on the spec file of Jason Petersen @ Citus. diff --git a/python/tests/files/debian.changelog.refer b/python/tests/files/debian.changelog.refer new file mode 100644 index 00000000..58d1e481 --- /dev/null +++ b/python/tests/files/debian.changelog.refer @@ -0,0 +1,1767 @@ +citus (10.0.2.citus-1) stable; urgency=low + + * Adds a configure flag to enforce security + + * Fixes a bug due to cross join without target list + + * Fixes a bug with UNION ALL on PG 13 + + * Fixes a compatibility issue with pg_audit in utility calls + + * Fixes insert query with CTEs/sublinks/subqueries etc + + * Grants SELECT permission on citus_tables view to public + + * Grants SELECT permission on columnar metadata tables to public + + * Improves citus_update_table_statistics and provides distributed deadlock + detection + + * Preserves colocation with procedures in alter_distributed_table + + * Prevents using alter_columnar_table_set and alter_columnar_table_reset + on a columnar table not owned by the user + + * Removes limits around long table names + + -- Gurkan Indibay Thu, 4 Mar 2021 2:46:54 +0000 + +citus (9.5.2.citus-1) stable; urgency=low + + * Fixes distributed deadlock detection being blocked by metadata sync + + * Prevents segfaults when SAVEPOINT handling cannot recover from connection + failures + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Wed, 27 Jan 2021 11:25:38 +0000 + +citus (9.4.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause router queries with local tables to be pushed + down + + * Fixes a segfault in connection management due to invalid connection hash + entries + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Tue, 5 Jan 2021 14:58:56 +0000 + +citus (9.5.1.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that could cause excessive memory consumption when a partition is + created + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + * Fixes a segfault in connection management due to invalid connection hash + entries + + -- Onur Tirtir Wed, 2 Dec 2020 14:28:44 +0000 + +citus (9.4.3.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + -- Onur Tirtir Tue, 24 Nov 2020 11:17:57 +0000 + +citus (9.5.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 13 + + * Removes the task-tracker executor + + * Introduces citus local tables + + * Introduces undistribute_table UDF to convert tables back to postgres tables + + * Adds support for EXPLAIN (ANALYZE) EXECUTE and EXPLAIN EXECUTE + + * Adds support for EXPLAIN (ANALYZE, WAL) for PG13 + + * Sorts the output of EXPLAIN (ANALYZE) by execution duration. + + * Adds support for CREATE TABLE ... USING table_access_method + + * Adds support for WITH TIES option in SELECT and INSERT SELECT queries + + * Avoids taking multi-shard locks on workers + + * Enforces citus.max_shared_pool_size config in COPY queries + + * Enables custom aggregates with multiple parameters to be executed on workers + + * Enforces citus.max_intermediate_result_size in local execution + + * Improves cost estimation of INSERT SELECT plans + + * Introduces delegation of procedures that read from reference tables + + * Prevents pull-push execution for simple pushdownable subqueries + + * Improves error message when creating a foreign key to a local table + + * Makes citus_prepare_pg_upgrade idempotent by dropping transition tables + + * Disallows ON TRUE outer joins with reference & distributed tables when + reference table is outer relation to avoid incorrect results + + * Disallows field indirection in INSERT/UPDATE queries to avoid incorrect + results + + * Disallows volatile functions in UPDATE subqueries to avoid incorrect results + + * Fixes CREATE INDEX CONCURRENTLY crash with local execution + + * Fixes citus_finish_pg_upgrade to drop all backup tables + + * Fixes a bug that cause failures when RECURSIVE VIEW joined reference table + + * Fixes DROP SEQUENCE failures when metadata syncing is enabled + + * Fixes a bug that caused CREATE TABLE with CHECK constraint to fail + + * Fixes a bug that could cause VACUUM to deadlock + + * Fixes master_update_node failure when no background worker slots are available + + * Fixes a bug that caused replica identity to not be propagated on shard repair + + * Fixes a bug that could cause crashes after connection timeouts + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug that could cause deadlocks on CREATE INDEX + + * Fixes a bug with genetic query optimization in outer joins + + * Fixes a crash when aggregating empty tables + + * Fixes a crash with inserting domain constrained composite types + + * Fixes a crash with multi-row & router INSERT's in local execution + + * Fixes a possibility of doing temporary file cleanup more than once + + * Fixes incorrect setting of join related fields + + * Fixes memory issues around deparsing index commands + + * Fixes reference table access tracking for sequential execution + + * Fixes removal of a single node with only reference tables + + * Fixes sending commands to coordinator when it is added as a worker + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlock + + -- Onur Tirtir Wed, 11 Nov 2020 15:00:27 +0000 + +citus (9.4.2.citus-1) stable; urgency=low + + * Fixes a bug that could lead to multiple maintenance daemons + + * Fixes an issue preventing views in reference table modifications + + -- Onur Tirtir Thu, 22 Oct 2020 8:53:44 +0000 + +citus (9.4.1.citus-1) stable; urgency=low + + * Fixes EXPLAIN ANALYZE output truncation + + * Fixes a deadlock during transaction recovery + + -- Onur Tirtir Wed, 30 Sep 2020 9:33:46 +0000 + +citus (9.4.0.citus-1) stable; urgency=low + + * Improves COPY by honoring max_adaptive_executor_pool_size config + + * Adds support for insert into local table select from distributed table + + * Adds support to partially push down tdigest aggregates + + * Adds support for receiving binary encoded results from workers using + citus.enable_binary_protocol + + * Enables joins between local tables and CTEs + + * Adds showing query text in EXPLAIN output when explain verbose is true + + * Adds support for showing CTE statistics in EXPLAIN ANALYZE + + * Adds support for showing amount of data received in EXPLAIN ANALYZE + + * Introduces downgrade paths in migration scripts + + * Avoids returning incorrect results when changing roles in a transaction + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes a bug that occurs when coordinator is added as a worker node + + * Fixes a crash because of overflow in partition id with certain compile flags + + * Fixes a crash that may happen if no worker nodes are added + + * Fixes a crash that occurs when inserting implicitly coerced constants + + * Fixes a crash when aggregating empty tables + + * Fixes a memory leak in subtransaction memory handling + + * Fixes crash when using rollback to savepoint after cancellation of DML + + * Fixes deparsing for queries with anonymous column references + + * Fixes distribution of composite types failing to include typemods + + * Fixes explain analyze on adaptive executor repartitions + + * Fixes possible error throwing in abort handle + + * Fixes segfault when evaluating func calls with default params on coordinator + + * Fixes several EXPLAIN ANALYZE issues + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlocks + + * Reports correct INSERT/SELECT method in EXPLAIN + + * Disallows triggers on citus tables + + -- Onur Tirtir Tue, 28 Jul 2020 13:22:31 +0000 + +citus (9.3.5.citus-1) stable; urgency=low + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a crash because of overflow in partition id with certain compile flags + + -- Onur Tirtir Mon, 27 Jul 2020 7:28:18 +0000 + +citus (9.3.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes deparsing for queries with anonymous column references + + -- Onur Tirtir Wed, 22 Jul 2020 9:00:01 +0000 + +citus (9.3.3.citus-1) stable; urgency=low + + * Fixes a memory leak in subtransaction memory handling + + -- Onur Tirtir Mon, 13 Jul 2020 8:47:40 +0000 + +citus (9.3.0.citus-1) stable; urgency=low + + * Adds max_shared_pool_size to control number of connections across sessions + + * Adds support for window functions on coordinator + + * Improves shard pruning logic to understand OR-conditions + + * Prevents using an extra connection for intermediate result multi-casts + + * Adds propagation of ALTER ROLE .. SET statements + + * Adds update_distributed_table_colocation UDF to update colocation of tables + + * Introduces a UDF to truncate local data after distributing a table + + * Adds support for creating temp schemas in parallel + + * Adds support for evaluation of nextval in the target list on coordinator + + * Adds support for local execution of COPY/TRUNCATE/DROP/DDL commands + + * Adds support for local execution of shard creation + + * Uses local execution in a transaction block + + * Adds support for querying distributed table sizes concurrently + + * Allows master_copy_shard_placement to replicate placements to new nodes + + * Allows table type to be used in target list + + * Avoids having multiple maintenance daemons active for a single database + + * Defers reference table replication to shard creation time + + * Enables joins between local tables and reference tables in transaction blocks + + * Ignores pruned target list entries in coordinator plan + + * Improves SIGTERM handling of maintenance daemon + + * Increases the default of citus.node_connection_timeout to 30 seconds + + * Fixes a bug that occurs when creating remote tasks in local execution + + * Fixes a bug that causes some DML queries containing aggregates to fail + + * Fixes a bug that could cause failures in queries with subqueries or CTEs + + * Fixes a bug that may cause some connection failures to throw errors + + * Fixes a bug which caused queries with SRFs and function evalution to fail + + * Fixes a bug with generated columns when executing COPY dist_table TO file + + * Fixes a crash when using non-constant limit clauses + + * Fixes a failure when composite types used in prepared statements + + * Fixes a possible segfault when dropping dist. table in a transaction block + + * Fixes a possible segfault when non-pushdownable aggs are solely used in HAVING + + * Fixes a segfault when executing queries using GROUPING + + * Fixes an error when using LEFT JOIN with GROUP BY on primary key + + * Fixes an issue with distributing tables having generated cols not at the end + + * Fixes automatic SSL permission issue when using "initdb --allow-group-access" + + * Fixes errors which could occur when subqueries are parameters to aggregates + + * Fixes possible issues by invalidating the plan cache in master_update_node + + * Fixes timing issues which could be caused by changing system clock + + -- Onur Tirtir Thu, 7 May 2020 15:11:25 +0000 + +citus (9.2.4.citus-1) stable; urgency=low + + * Fixes a release problem in 9.2.3 + + -- Onur Tirtir Tue, 31 Mar 2020 08:06:59 +0000 + +citus (9.2.3.citus-1) stable; urgency=low + + * Do not use C functions that have been banned by Microsoft + + * Fixes a bug that causes wrong results with complex outer joins + + * Fixes issues found using static analysis + + * Fixes left join shard pruning in pushdown planner + + * Fixes possibility for segmentation fault in internal aggregate functions + + * Fixes possible segfault when non pushdownable aggregates are used in HAVING + + * Improves correctness of planning subqueries in HAVING + + * Prevents using old connections for security if citus.node_conninfo changed + + * Uses Microsoft approved cipher string for default TLS setup + + -- Onur Tirtir Thu, 26 Mar 2020 8:22:48 +0000 + +citus (9.0.2.citus-1) stable; urgency=low + + * Fixes build errors on EL/OL 6 based distros + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Disallows distributed function creation when replication_model is `statement` + + -- Onur Tirtir Fri, 6 Mar 2020 14:10:16 +0000 + +citus (9.2.2.citus-1) stable; urgency=low + + * Fixes a bug that caused some prepared stmts with function calls to fail + + * Fixes a bug that caused some prepared stmts with composite types to fail + + * Fixes a bug that caused missing subplan results in workers + + * Improves performance of re-partition joins + + -- Onur Tirtir Fri, 6 Mar 2020 07:14:20 +0000 + +citus (9.2.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes if distribution key is NULL + + -- Onur Tirtir Fri, 14 Feb 2020 11:51:09 +0000 + +citus (9.2.0.citus-1) stable; urgency=low + + * Adds support for INSERT...SELECT queries with re-partitioning + + * Adds citus.coordinator_aggregation_strategy to support more aggregates + + * Adds caching of local plans on shards for Citus MX + + * Adds compatibility support for dist. object infrastructure from old versions + + * Adds defering shard-pruning for fast-path router queries to execution + + * Adds propagation of GRANT ... ON SCHEMA queries + + * Adds support for CTE pushdown via CTE inlining in distributed planning + + * Adds support for ALTER TABLE ... SET SCHEMA propagation. + + * Adds support for DROP ROUTINE & ALTER ROUTINE commands + + * Adds support for any inner join on a reference table + + * Changes citus.log_remote_commands level to NOTICE + + * Disallows marking ref. table shards unhealthy in the presence of savepoints + + * Disallows placing new shards with shards in TO_DELETE state + + * Enables local execution of queries that do not need any data access + + * Fixes Makefile trying to cleanup PG directory during install + + * Fixes a bug causing errors when planning a query with multiple subqueries + + * Fixes a possible deadlock that could happen during shard moves + + * Fixes a problem when adding a new node due to tables referenced in func body + + * Fixes an issue that could cause joins with reference tables to be slow + + * Fixes cached metadata for shard is inconsistent issue + + * Fixes inserting multiple composite types as partition key in VALUES + + * Fixes unnecessary repartition on joins with more than 4 tables + + * Prevents wrong results for replicated partitioned tables after failure + + * Restricts LIMIT approximation for non-commutative aggregates + + -- Onur Tirtir Wed, 10 Feb 2020 8:48:00 +0000 + +citus (9.1.1.citus-1) stable; urgency=low + + * Fixes a bug causing SQL-executing UDFs to crash when passing in DDL + + * Fixes a bug that caused column_to_column_name to crash for invalid input + + * Fixes a bug that caused inserts into local tables w/ dist. subqueries to crash + + * Fixes a bug that caused some noop DML statements to fail + + * Fixes a bug that prevents dropping reference table columns + + * Fixes a crash in IN (.., NULL) queries + + * Fixes a crash when calling a distributed function from PL/pgSQL + + * Fixes an issue that caused CTEs to sometimes leak connections + + * Fixes strange errors in DML with unreachable sublinks + + * Prevents statements in SQL functions to run outside of a transaction + + -- Onur Tirtir Wed, 18 Dec 2019 14:32:42 +0000 + +citus (9.1.0.citus-1) stable; urgency=low + + * Adds extensions to distributed object propagation infrastructure + + * Adds support for ALTER ROLE propagation + + * Adds support for aggregates in create_distributed_function + + * Adds support for expressions in reference joins + + * Adds support for returning RECORD in multi-shard queries + + * Adds support for simple IN subqueries on unique cols in repartition joins + + * Adds support for subqueries in HAVING clauses + + * Automatically distributes unary aggs w/ combinefunc and non-internal stype + + * Disallows distributed func creation when replication_model is 'statement' + + * Drops support for deprecated real-time and router executors + + * Fixes a bug in local execution that could cause missing rows in RETURNING + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Fixes a bug that caused other CREATE EXTENSION commands to take longer + + * Fixes a bug that prevented REFRESH MATERIALIZED VIEW + + * Fixes a bug when view is used in modify statements + + * Fixes a memory leak in adaptive executor when query returns many columns + + * Fixes underflow init of default values in worker extended op node creation + + * Fixes potential segfault in standard_planner inlining functions + + * Fixes an issue that caused failures in RHEL 6 builds + + * Fixes queries with repartition joins and group by unique column + + * Improves CTE/Subquery performance by pruning intermediate rslt broadcasting + + * Removes citus.worker_list_file GUC + + * Revokes usage from the citus schema from public + + -- Onur Tirtir Thu, 28 Nov 2019 15:11:05 +0000 + +citus (9.0.1.citus-1) stable; urgency=low + + * Fixes a memory leak in the executor + + * Revokes usage from the citus schema from public + + -- Hanefi Onaldi Wed, 30 Oct 2019 8:53:22 +0000 + +citus (9.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 12 + + * Adds UDFs to help with PostgreSQL upgrades + + * Distributes types to worker nodes + + * Introduces create_distributed_function UDF + + * Introduces local query execution for Citus MX + + * Implements infrastructure for routing CALL to MX workers + + * Implements infrastructure for routing SELECT function() to MX workers + + * Adds support for foreign key constraints between reference tables + + * Adds a feature flag to turn off CREATE TYPE propagation + + * Adds option citus.single_shard_commit_protocol + + * Adds support for EXPLAIN SUMMARY + + * Adds support for GENERATE ALWAYS AS STORED + + * Adds support for serial and smallserial in MX mode + + * Adds support for anon composite types on the target list in router queries + + * Avoids race condition between create_reference_table & master_add_node + + * Fixes a bug in schemas of distributed sequence definitions + + * Fixes a bug that caused run_command_on_colocated_placements to fail + + * Fixes a bug that leads to various issues when a connection is lost + + * Fixes a schema leak on CREATE INDEX statement + + * Fixes assert failure in bare SELECT FROM reference table FOR UPDATE in MX + + * Makes master_update_node MX compatible + + * Prevents pg_dist_colocation from multiple records for reference tables + + * Prevents segfault in worker_partition_protocol edgecase + + * Propagates ALTER FUNCTION statements for distributed functions + + * Propagates CREATE OR REPLACE FUNCTION for distributed functions + + * Propagates REINDEX on tables & indexes + + * Provides a GUC to turn of the new dependency propagation functionality + + * Uses 2PC in adaptive executor when dealing with replication factors above 1 + + -- Hanefi Onaldi Tue, 15 Oct 2019 16:54:50 +0000 + +citus (8.3.2.citus-1) stable; urgency=low + + * Fixes performance issues by skipping unnecessary relation access recordings + + -- Hanefi Onaldi Fri, 9 Aug 2019 11:15:57 +0000 + +citus (8.3.1.citus-1) stable; urgency=low + + * Improves Adaptive Executor performance + + -- Hanefi Onaldi Mon, 29 Jul 2019 10:25:50 +0000 + +citus (8.3.0.citus-1) stable; urgency=low + + * Adds a new distributed executor: Adaptive Executor + + * citus.enable_statistics_collection defaults to off (opt-in) + + * Adds support for CTEs in router planner for modification queries + + * Adds support for propagating SET LOCAL at xact start + + * Adds option to force master_update_node during failover + + * Deprecates master_modify_multiple_shards + + * Improves round robin logic on router queries + + * Creates all distributed schemas as superuser on a separate connection + + * Makes COPY adapt to connection use behaviour of previous commands + + * Replaces SESSION_LIFESPAN with configurable no. of connections at xact end + + * Propagates ALTER FOREIGN TABLE commands to workers + + * Don't schedule tasks on inactive nodes + + * Makes DROP/VALIDATE CONSTRAINT tolerant of ambiguous shard extension + + * Fixes an issue with subquery map merge jobs as non-root + + * Fixes null pointers caused by partial initialization of ConnParamsHashEntry + + * Fixes errors caused by joins with shadowed aliases + + * Fixes a regression in outer joining subqueries introduced in 8.2.0 + + * Fixes a crash that can occur under high memory load + + * Fixes a bug that selects wrong worker when using round-robin assignment + + * Fixes savepoint rollback after multi-shard modify/copy failure + + * Fixes bad foreign constraint name search + + * Fixes a bug that prevents stack size to be adjusted + + -- Hanefi Onaldi Wed, 10 Jul 2019 15:19:02 +0000 + +citus (8.2.2.citus-1) stable; urgency=low + + * Fixes a bug in outer joins wrapped in subqueries + + -- Burak Velioglu Wed, 12 Jun 2019 8:45:08 +0000 + +citus (8.2.1.citus-1) stable; urgency=low + + * Fixes a bug that prevents stack size to be adjusted + + -- Burak Velioglu Wed, 3 Apr 2019 20:56:47 +0000 + +citus (8.1.2.citus-1) stable; urgency=low + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Fixes a bug that prevents stack size to be adjusted + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + -- Burak Velioglu Wed, 3 Apr 2019 20:34:46 +0000 + +citus (8.2.0.citus-1) stable; urgency=low + + * Removes support and code for PostgreSQL 9.6 + + * Enable more outer joins with reference tables + + * Execute CREATE INDEX CONCURRENTLY in parallel + + * Treat functions as transaction blocks + + * Add support for column aliases on join clauses + + * Skip standard_planner() for trivial queries + + * Added support for function calls in joins + + * Round-robin task assignment policy relies on local transaction id + + * Relax subquery union pushdown restrictions for reference tables + + * Speed-up run_command_on_shards() + + * Address some memory issues in connection config + + * Restrict visibility of get_*_active_transactions functions to pg_monitor + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Queries with only intermediate results do not rely on task assignment policy + + * Finish connection establishment in parallel for multiple connections + + * Fixes a bug related to pruning shards using a coerced value + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + * Fixes a bug that could lead to infinite recursion during recursive planning + + * Fixes a bug that could prevent planning full outer joins with using clause + + * Fixes a bug that could lead to memory leak on citus_relation_size + + * Fixes a problem that could cause segmentation fault with recursive planning + + * Switch CI solution to CircleCI + + -- Burak Velioglu Fri, 29 Mar 2019 07:36:09 +0000 + +citus (8.0.3.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Wed, 9 Jan 2019 9:50:07 +0000 + +citus (8.1.1.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Mon, 7 Jan 2019 16:26:13 +0000 + +citus (8.1.0.citus-1) stable; urgency=low + + * Turns on ssl by default for new installations of citus + + * Restricts SSL Ciphers to TLS1.2 and above + + * Adds support for INSERT INTO SELECT..ON CONFLICT/RETURNING via coordinator + + * Adds support for round-robin task assignment for queries to reference tables + + * Adds support for SQL tasks using worker_execute_sql_task UDF with task-tracker + + * Adds support for VALIDATE CONSTRAINT queries + + * Adds support for disabling hash aggregate with HLL + + * Adds user ID suffix to intermediate files generated by task-tracker + + * Only allow transmit from pgsql_job_cache directory + + * Disallows GROUPING SET clauses in subqueries + + * Removes restriction on user-defined group ID in node addition functions + + * Relaxes multi-shard modify locks when enable_deadlock_prevention is disabled + + * Improves security in task-tracker protocol + + * Improves permission checks in internal DROP TABLE functions + + * Improves permission checks in cluster management functions + + * Cleans up UDFs and fixes permission checks + + * Fixes crashes caused by stack size increase under high memory load + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Tue, 18 Dec 2018 15:12:45 +0000 + +citus (8.0.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Thu, 13 Dec 2018 13:56:44 +0000 + +citus (7.5.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Wed, 12 Dec 2018 11:45:24 +0000 + +citus (8.0.1.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 11:38:47 +0000 + +citus (7.5.3.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 10:52:20 +0000 + +citus (7.5.2.citus-1) stable; urgency=low + + * Fixes inconsistent metadata error when shard metadata caching get interrupted + + * Fixes a bug that could cause memory leak + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Wed, 14 Nov 2018 20:42:16 +0000 + +citus (8.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 11 + + * Adds support for applying DML operations on reference tables from MX nodes + + * Adds distributed locking to truncated MX tables + + * Adds support for running TRUNCATE command from MX worker nodes + + * Adds views to provide insight about the distributed transactions + + * Adds support for TABLESAMPLE in router queries + + * Adds support for INCLUDE option in index creation + + * Adds option to allow simple DML commands from hot standby + + * Adds support for partitioned tables with replication factor > 1 + + * Prevents a deadlock on concurrent DROP TABLE and SELECT on Citus MX + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes a bug in MX mode, calling DROP SCHEMA with existing partitioned table + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + * Fixes a bug preventing rollback in CREATE PROCEDURE + + * Fixes a bug on not being able to drop index on a partitioned table + + * Fixes a bug on TRUNCATE when there is a foreign key to a reference table + + * Fixes a performance issue in prepared INSERT..SELECT + + * Fixes a bug which causes errors on DROP DATABASE IF EXISTS + + * Fixes a bug to remove intermediate result directory in pull-push execution + + * Improves query pushdown planning performance + + * Evaluate functions anywhere in query + + -- Burak Velioglu Fri, 02 Nov 2018 08:06:42 +0000 + +citus (7.5.1.citus-1) stable; urgency=low + + * Improves query pushdown planning performance + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + -- Burak Velioglu Wed, 29 Aug 2018 08:06:42 +0000 + +citus (7.4.2.citus-1) stable; urgency=low + + * Fixes a segfault in real-time executor during online shard move + + -- Mehmet Furkan Sahin Fri, 27 Jul 2018 13:42:27 +0000 + +citus (7.5.0.citus-1) stable; urgency=low + + * Adds foreign key support from hash distributed to reference tables + + * Adds SELECT ... FOR UPDATE support for router plannable queries + + * Adds support for non-partition columns in count distinct + + * Fixes a segfault in real-time executor during online shard move + + * Fixes ALTER TABLE ADD COLUMN constraint check + + * Fixes a bug where INSERT ... SELECT allows one to update dist. column + + * Allows DDL commands to be sequentialized via citus.multi_shard_modify_mode + + * Adds support for topn_union_agg and topn_add_agg across shards + + * Adds support for hll_union_agg and hll_add_agg across shards + + * Fixes a bug that might cause shards to have a wrong owner + + * Adds select_opens_transaction_block GUC + + * Adds utils to implement DDLs for policies in future + + * Makes intermediate results to use separate connections + + * Adds a node_conninfo GUC to set outgoing connection settings + + -- Mehmet Furkan Sahin Wed, 25 Jul 2018 9:32:24 +0000 + +citus (6.2.6.citus-1) stable; urgency=low + + * Adds support for respecting enable_hashagg in the master planner + + -- Burak Velioglu Fri, 06 Jul 2018 13:30:08 +0000 + +citus (7.4.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause txns to incorrectly proceed after failure + + * Fixes a bug on INSERT ... SELECT queries in prepared statements + + -- Burak Velioglu Wed, 20 Jun 2018 12:25:30 +0000 + +citus (7.2.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause SELECTs to crash during a rebalance + + -- Burak Velioglu Thu, 17 May 2018 11:51:56 +0000 + +citus (7.4.0.citus-1) stable; urgency=low + + * Adds support for non-pushdownable subqueries and CTEs in UPDATE/DELETE + + * Adds support for pushdownable subqueries and joins in UPDATE/DELETE + + * Adds faster shard pruning for subqueries + + * Adds partitioning support to MX table + + * Adds support for (VACUUM | ANALYZE) VERBOSE + + * Adds support for multiple ANDs in HAVING for pushdown planner + + * Adds support for quotation needy schema names + + * Improves operator check time in physical planner for custom data types + + * Removes broadcast join logic + + * Deprecates large_table_shard_count and master_expire_table_cache() + + * Modifies master_update_node to write-lock shards hosted by node over update + + * DROP TABLE now drops shards as the currrent user instead of the superuser + + * Adds specialised error codes for connection failures + + * Improves error messages on connection failure + + * Fixes issue which prevented multiple citus_table_size calls per query + + * Tests are updated to use create_distributed_table + + -- Burak Velioglu Tue, 15 May 2018 13:01:17 +0000 + +citus (7.3.0.citus-1) stable; urgency=low + + * Adds support for non-colocated joins between subqueries + + * Adds support for window functions that can be pushed down to worker + + * Adds support for modifying CTEs + + * Adds recursive plan for WHERE clause subqueries with recurring FROM clause + + * Adds support for bool_ and bit_ aggregates + + * Adds support for Postgres jsonb and json aggregation functions + + * Adds support for respecting enable_hashagg in the master plan + + * Performance improvements to reduce distributed planning time + + * Fixes a bug on planner when aggregate is used in ORDER BY + + * Fixes a bug on planner when DISTINCT (ON) clause is used with GROUP BY + + * Fixes a planner bug with distinct and aggregate clauses + + * Fixes a bug that opened new connections on each table size function call + + * Fixes a bug canceling backends not involved in distributed deadlocks + + * Fixes count distinct bug on column expressions when used with subqueries + + * Improves error handling on worker node failures + + * Improves error messages for INSERT queries that have subqueries + + -- Burak Velioglu Thu, 15 Mar 2018 14:16:10 +0000 + +citus (7.2.1.citus-1) stable; urgency=low + + * Fixes count distinct bug on column expressions when used with subqueries + + * Adds support for respecting enable_hashagg in the master plan + + * Fixes a bug canceling backends not involved in distributed deadlocks + + -- Burak Velioglu Tue, 06 Feb 2018 14:46:07 +0000 + +citus (7.2.0.citus-1) stable; urgency=low + + * Adds support for CTEs + + * Adds support for subqueries that require merge step + + * Adds support for set operations (UNION, INTERSECT, ...) + + * Adds support for 2PC auto-recovery + + * Adds support for querying local tables in CTEs and subqueries + + * Adds support for more SQL coverage in subqueries for reference tables + + * Adds support for count(distinct) in queries with a subquery + + * Adds support for non-equijoins when there is already an equijoin + + * Adds support for real-time executor to run in transaction blocks + + * Adds infrastructure for storing intermediate distributed query results + + * Adds a new GUC named enable_repartition_joins for auto executor switch + + * Adds support for limiting the intermediate result size + + * Improves support for queries with unions containing filters + + * Improves support for queries with unions containing joins + + * Improves support for subqueries in the WHERE clause + + * Increases COPY throughput + + * Enables pushing down queries containing only recurring tuples and GROUP BY + + * Load-balance queries that read from 0 shards + + * Improves support for using functions in subqueries + + * Fixes a bug that causing real-time executor to crash during cancellation + + * Fixes a bug that causing real-time executor to get stuck on cancellation + + * Fixes a bug that could block modification queries unnecessarily + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Fixes a bug that could cause an assert failure with ANALYZE statements + + * Fixes a bug that would push down wrong set operations in subqueries + + * Fixes a bug that could cause a deadlock in create_distributed_table + + * Fixes a bug that could confuse user about ANALYZE usage + + * Fixes a bug causing false positive distributed deadlock detections + + * Relaxes the locking for DDL commands on partitioned tables + + * Relaxes the locking on COPY with replication + + * Logs more remote commands when citus.log_remote_commands is set + + -- Burak Velioglu Tue, 16 Jan 2018 14:34:20 +0000 + +citus (6.2.5.citus-1) stable; urgency=low + + * Fixes a bug that could crash the coordinator while reporting a remote error + + -- Burak Velioglu Thu, 11 Jan 2018 11:40:28 +0000 + +citus (7.1.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Increases COPY throughput + + -- Burak Velioglu Fri, 05 Jan 2018 09:00:07 +0000 + +citus (7.1.1.citus-1) stable; urgency=low + + * Fixes a bug preventing pushing down subqueries with reference tables + + * Fixes a bug that could create false positive distributed deadlocks + + * Fixes a bug that could prevent running concurrent COPY and multi-shard DDL + + * Fixes a bug that could mislead users about ANALYZE queries + + -- Burak Velioglu Tue, 05 Dec 2017 09:00:07 +0000 + +citus (7.1.0.citus-1) stable; urgency=low + + * Adds support for native queries with multi shard UPDATE/DELETE queries + + * Expands reference table support in subquery pushdown + + * Adds window function support for subqueries and INSERT ... SELECT queries + + * Adds support for COUNT(DISTINCT) [ON] queries on non-partition columns + + * Adds support for DISTINCT [ON] queries on non-partition columns + + * Introduces basic usage statistic collector + + * Adds support for setting replica identity while creating distributed tables + + * Adds support for ALTER TABLE ... REPLICA IDENTITY queries + + * Adds pushdown support for LIMIT and HAVING grouped by partition key + + * Adds support for INSERT ... SELECT queries via worker nodes on MX clusters + + * Adds support for adding primary key using already defined index + + * Adds replication parameter to shard copy functions + + * Changes shard_name UDF to omit public schema name + + * Adds master_move_node UDF to make changes on nodename/nodeport more easy + + * Fixes a bug that could cause casting error with INSERT ... SELECT queries + + * Fixes a bug that could prevent upgrading servers from Citus 6.1 + + * Fixes a bug that could prevent attaching partitions to a table in schema + + * Fixes a bug preventing adding nodes to clusters with reference tables + + * Fixes a bug that could cause a crash with INSERT ... SELECT queries + + * Fixes a bug that could prevent creating a partitoned table on Cloud + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Velioglu Wed, 15 Nov 2017 09:00:07 +0000 + +citus (7.0.3.citus-1) stable; urgency=low + + * Fixes several bugs that could cause crash + + * Fixes a bug that could cause deadlock while creating reference tables + + * Fixes a bug that could cause false-positives in deadlock detection + + * Fixes a bug that could cause 2PC recovery not to work from MX workers + + * Fixes a bug that could cause cache incohorency + + * Fixes a bug that could cause maintenance daemon to skip cache invalidations + + * Improves performance of transaction recovery by using correct index + + -- Burak Yucesoy Mon, 16 Oct 2017 11:52:07 +0000 + +citus (7.0.2.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 22:29:01 +0000 + +citus (6.2.4.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 21:31:35 +0000 + +citus (6.1.3.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 20:31:35 +0000 + +citus (7.0.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause memory leaks in INSERT ... SELECT queries + + * Fixes a bug that could cause incorrect execution of prepared statements + + * Fixes a bug that could cause excessive memory usage during COPY + + * Incorporates latest changes from core PostgreSQL code + + -- Burak Yucesoy Tue, 12 Sep 2017 17:53:50 +0000 + +citus (7.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 10 + + * Drops support for PostgreSQL 9.5 + + * Adds support for multi-row INSERT + + * Adds support for router UPDATE and DELETE queries with subqueries + + * Adds infrastructure for distributed deadlock detection + + * Deprecates enable_deadlock_prevention flag + + * Adds support for partitioned tables + + * Adds support for creating UNLOGGED tables + + * Adds support for SAVEPOINT + + * Adds UDF citus_create_restore_point for taking distributed snapshots + + * Adds support for evaluating non-pushable INSERT ... SELECT queries + + * Adds support for subquery pushdown on reference tables + + * Adds shard pruning support for IN and ANY + + * Adds support for UPDATE and DELETE commands that prune down to 0 shard + + * Enhances transaction support by relaxing some transaction restrictions + + * Fixes a bug causing crash if distributed table has no shards + + * Fixes a bug causing crash when removing inactive node + + * Fixes a bug causing failure during COPY on tables with dropped columns + + * Fixes a bug causing failure during DROP EXTENSION + + * Fixes a bug preventing executing VACUUM and INSERT concurrently + + * Fixes a bug in prepared INSERT statements containing an implicit cast + + * Fixes several issues related to statement cancellations and connections + + * Fixes several 2PC related issues + + * Removes an unnecessary dependency causing warning messages in pg_dump + + * Adds internal infrastructure for follower clusters + + * Adds internal infrastructure for progress tracking + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Mon, 28 Aug 2017 12:27:53 +0000 + +citus (6.2.3.citus-1) stable; urgency=low + + * Fixes a crash during execution of local CREATE INDEX CONCURRENTLY + + * Fixes a bug preventing usage of quoted column names in COPY + + * Fixes a bug in prepared INSERTs with implicit cast in partition column + + * Relaxes locks in VACUUM to ensure concurrent execution with INSERT + + -- Burak Yucesoy Thu, 13 Jul 2017 11:27:17 +0000 + +citus (6.2.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Burak Velioglu Wed, 07 Jun 2017 09:42:17 +0000 + +citus (6.1.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Jason Petersen Wed, 31 May 2017 16:14:11 +0000 + +citus (6.2.1.citus-1) stable; urgency=low + + * Relaxes version-check logic to avoid breaking non-distributed commands + + -- Jason Petersen Wed, 24 May 2017 22:36:07 +0000 + +citus (6.2.0.citus-1) stable; urgency=low + + * Increases SQL subquery coverage by pushing down more kinds of queries + + * Adds CustomScan API support to allow read-only transactions + + * Adds support for CREATE/DROP INDEX CONCURRENTLY + + * Adds support for ALTER TABLE ... ADD CONSTRAINT + + * Adds support for ALTER TABLE ... RENAME COLUMN + + * Adds support for DISABLE/ENABLE TRIGGER ALL + + * Adds support for expressions in the partition column in INSERTs + + * Adds support for query parameters in combination with function evaluation + + * Adds support for creating distributed tables from non-empty local tables + + * Adds UDFs to get size of distributed tables + + * Adds UDFs to add a new node without replicating reference tables + + * Adds checks to prevent running Citus binaries with wrong metadata tables + + * Improves shard pruning performance for range queries + + * Improves planner performance for joins involving co-located tables + + * Improves shard copy performance by creating indexes after copy + + * Improves task-tracker performance by batching several status checks + + * Enables router planner for queries on range partitioned table + + * Changes TRUNCATE to drop local data only if enable_ddl_propagation is off + + * Starts to execute DDL on coordinator before workers + + * Fixes a bug causing incorrectly reading invalidated cache + + * Fixes a bug related to creation of schemas in workers with incorrect owner + + * Fixes a bug related to concurrent run of shard drop functions + + * Fixes a bug related to EXPLAIN ANALYZE with DML queries + + * Fixes a bug related to SQL functions in FROM clause + + * Adds a GUC variable to report cross shard queries + + * Fixes a bug related to partition columns without native hash function + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Tue, 16 May 2017 16:05:22 +0000 + +citus (6.1.1.citus-1) stable; urgency=low + + * Fixes a crash caused by router executor use after connection timeouts + + * Fixes a crash caused by relation cache invalidation during COPY + + * Fixes bug related to DDL use within PL/pgSQL functions + + * Fixes a COPY bug related to types lacking binary output functions + + * Fixes a bug related to modifications with parameterized partition values + + * Fixes improper value interpolation in worker sequence generation + + * Guards shard pruning logic against zero-shard tables + + * Fixes possible NULL pointer dereference and buffer underflow, via PVS-Studio + + * Fixes a INSERT...SELECT bug that could push down non-partition column JOINs + + -- Metin Doslu Fri, 5 May 2017 17:42:00 +0000 + +citus (6.1.0.citus-1) stable; urgency=low + + * Implements reference tables, transactionally replicated to all nodes + + * Adds upgrade_to_reference_table UDF to upgrade pre-6.1 reference tables + + * Expands prepared statement support to nearly all statements + + * Adds support for creating VIEWs which reference distributed tables + + * Adds targeted VACUUM/ANALYZE support + + * Adds support for the FILTER clause in aggregate expressions + + * Adds support for function evaluation within INSERT INTO ... SELECT + + * Adds support for creating foreign key constraints with ALTER TABLE + + * Adds logic to choose router planner for all queries it supports + + * Enhances create_distributed_table with parameter for explicit colocation + + * Adds generally useful utility UDFs previously available as "Citus Tools" + + * Adds user-facing UDFs for locking shard resources and metadata + + * Refactors connection and transaction management for more consistency + + * Enhances COPY with fully transactional semantics + + * Improves support for cancellation for a number of queries and commands + + * Adds column_to_column_name UDF to help users understand partkey values + + * Adds master_disable_node UDF for temporarily disabling nodes + + * Adds proper MX ("masterless") metadata propagation logic + + * Adds start_metadata_sync_to_node UDF to propagate metadata changes to nodes + + * Enhances SERIAL compatibility with MX tables + + * Adds an node_connection_timeout parameter to set node connection timeouts + + * Adds enable_deadlock_prevention setting to permit multi-node transactions + + * Adds a replication_model setting to specify replication of new tables + + * Changes the shard_replication_factor setting's default value to one + + * Adds code to automatically set max_prepared_transactions if not configured + + * Accelerates lookup of colocated shard placements + + * Fixes a bug affecting INSERT INTO ... SELECT queries using constant values + + * Fixes a bug by ensuring COPY does not mark placements inactive + + * Fixes a bug affecting reads from pg_dist_shard_placement table + + * Fixes a crash triggered by creating a foreign key without a column + + * Fixes a crash related to accessing catalog tables after aborted transaction + + * Fixes a bug affecting JOIN queries requiring repartitions + + * Fixes a bug affecting node insertions to pg_dist_node table + + * Fixes a crash triggered by queries with modifying common table expressions + + * Fixes a bug affecting workloads with concurrent shard appends and deletions + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Thu, 9 Feb 2017 16:17:41 +0000 + +citus (6.0.1.citus-3) stable; urgency=low + + * First build using new versioning practices + + -- Jason Petersen Wed, 8 Feb 2017 23:19:46 +0000 + +citus (6.0.1.citus-2) stable; urgency=low + + * Transitional package to guide users to new package name + + -- Jason Petersen Mon, 6 Feb 2017 16:33:44 +0000 + +citus (6.0.1.citus-1) stable; urgency=low + + * Fixes a bug causing failures during pg_upgrade + + * Fixes a bug preventing DML queries during colocated table creation + + * Fixes a bug that caused NULL parameters to be incorrectly passed as text + + -- Burak Yucesoy Wed, 30 Nov 2016 15:27:38 +0000 + +citus (6.0.0.citus-1) stable; urgency=low + + * Adds compatibility with PostgreSQL 9.6, now the recommended version + + * Removes the pg_worker_list.conf file in favor of a pg_dist_node table + + * Adds master_add_node and master_add_node UDFs to manage membership + + * Removes the \stage command and corresponding csql binary in favor of COPY + + * Removes copy_to_distributed_table in favor of first-class COPY support + + * Adds support for multiple DDL statements within a transaction + + * Adds support for certain foreign key constraints + + * Adds support for parallel INSERT INTO ... SELECT against colocated tables + + * Adds support for the TRUNCATE command + + * Adds support for HAVING clauses in SELECT queries + + * Adds support for EXCLUDE constraints which include the partition column + + * Adds support for system columns in queries (tableoid, ctid, etc.) + + * Adds support for relation name extension within INDEX definitions + + * Adds support for no-op UPDATEs of the partition column + + * Adds several general-purpose utility UDFs to aid in Citus maintenance + + * Adds master_expire_table_cache UDF to forcibly expire cached shards + + * Parallelizes the processing of DDL commands which affect distributed tables + + * Adds support for repartition jobs using composite or custom types + + * Enhances object name extension to handle long names and large shard counts + + * Parallelizes the master_modify_multiple_shards UDF + + * Changes distributed table creation to error if target table is not empty + + * Changes the pg_dist_shard.logicalrelid column from an oid to regclass + + * Adds a placementid column to pg_dist_shard_placement, replacing Oid use + + * Removes the pg_dist_shard.shardalias distribution metadata column + + * Adds pg_dist_partition.repmodel to track tables using streaming replication + + * Adds internal infrastructure to take snapshots of distribution metadata + + * Addresses the need to invalidate prepared statements on metadata changes + + * Adds a mark_tables_colocated UDF for denoting pre-6.0 manual colocation + + * Fixes a bug affecting prepared statement execution within PL/pgSQL + + * Fixes a bug affecting COPY commands using composite types + + * Fixes a bug that could cause crashes during EXPLAIN EXECUTE + + * Separates worker and master job temporary folders + + * Eliminates race condition between distributed modification and repair + + * Relaxes the requirement that shard repairs also repair colocated shards + + * Implements internal functions to track which tables' shards are colocated + + * Adds pg_dist_partition.colocationid to track colocation group membership + + * Extends shard copy and move operations to respect colocation settings + + * Adds pg_dist_local_group to prepare for future MX-related changes + + * Adds create_distributed_table to easily create shards and infer colocation + + -- Jason Petersen Tue, 8 Nov 2016 19:45:45 +0000 + +citus (5.2.2.citus-1) stable; urgency=low + + * Adds support for IF NOT EXISTS clause of CREATE INDEX command + + * Adds support for RETURN QUERY and FOR ... IN PL/pgSQL features + + * Extends the router planner to handle more queries + + * Changes COUNT of zero-row sets to return 0 rather than an empty result + + * Reduces the minimum permitted task_tracker_delay to a single millisecond + + * Fixes a bug that caused crashes during joins with a WHERE false clause + + * Fixes a bug triggered by unique violation errors raised in long txns + + * Fixes a bug resulting in multiple registration of transaction callbacks + + * Fixes a bug which could result in stale reads of distribution metadata + + * Fixes a bug preventing distributed modifications in some PL/pgSQL functions + + * Fixes some code paths that could hypothetically read uninitialized memory + + * Lowers log level of "waiting for activity" messages + + -- Jason Petersen Tue, 8 Nov 2016 18:43:37 +0000 + +citus (5.2.1.citus-1) stable; urgency=low + + * Fixes subquery pushdown to properly extract outer join qualifiers + + * Addresses possible memory leak during multi-shard transactions + + -- Jason Petersen Tue, 6 Sep 2016 20:47:15 +0000 + +citus (5.2.0.citus-1) stable; urgency=low + + * Drops support for PostgreSQL 9.4; PostgreSQL 9.5 is required + + * Adds schema support for tables, named objects (types, operators, etc.) + + * Evaluates non-immutable functions on master in all modification commands + + * Adds support for SERIAL types in non-partition columns + + * Adds support for RETURNING clause in INSERT, UPDATE, and DELETE commands + + * Adds support for multi-statement transactions using a fixed set of nodes + + * Full SQL support for SELECT queries which can be executed on single worker + + * Adds option to perform DDL changes using prepared transactions (2PC) + + * Adds an enable_ddl_propagation parameter to control DDL propagation + + * Accelerates shard pruning during merges + + * Adds master_modify_multiple_shards UDF to modify many shards at once + + * Adds COPY support for arrays of user-defined types + + * Now supports parameterized prepared statements for certain use cases + + * Extends LIMIT/OFFSET support to all executor types + + * Constraint violations now fail fast rather than hitting all placements + + * Makes master_create_empty_shard aware of shard placement policy + + * Reduces unnecessary sleep during queries processed by real-time executor + + * Improves task tracker executor's task cleanup logic + + * Relaxes restrictions on cancellation of DDL commands + + * Removes ONLY keyword from worker SELECT queries + + * Error message improvements and standardization + + * Moves master_update_shard_statistics function to pg_catalog schema + + * Fixes a bug where hash-partitioned anti-joins could return bad results + + * Now sets storage type correctly for foreign table-backed shards + + * Fixes master_update_shard_statistics issue with hash-partitioned tables + + * Fixes an issue related to extending table names that require escaping + + * Reduces risk of row counter overflows during modifications + + * Fixes a crash related to FILTER clause use in COUNT DISTINCT subqueries + + * Fixes crashes related to partition columns with high attribute numbers + + * Fixes certain subquery and join crashes + + * Detects flex for build even if PostgreSQL was built without it + + * Fixes assert-enabled crash when all_modifications_commutative is true + + -- Jason Petersen Wed, 17 Aug 2016 10:23:21 +0000 + +citus (5.2.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.2. + + -- Jason Petersen Mon, 01 Aug 2016 17:01:05 +0000 + +citus (5.1.1-1) stable; urgency=low + + * Adds complex count distinct expression support in repartitioned subqueries + + * Improves task tracker job cleanup logic, addressing a memory leak + + * Fixes bug that generated incorrect results for LEFT JOIN queries + + * Improves compatibility with Debian's reproducible builds project + + * Fixes build issues on FreeBSD platforms + + -- Jason Petersen Fri, 17 Jun 2016 16:20:15 +0000 + +citus (5.1.0-1) stable; urgency=low + + * Adds distributed COPY to rapidly populate distributed tables + + * Adds support for using EXPLAIN on distributed queries + + * Recognizes and fast-paths single-shard SELECT statements automatically + + * Increases INSERT throughput via shard pruning optimizations + + * Improves planner performance for joins involving tables with many shards + + * Adds ability to pass columns as arguments to function calls in UPDATEs + + * Introduces transaction manager for use by multi-shard commands + + * Adds COUNT(DISTINCT ...) pushdown optimization for hash-partitioned tables + + * Adds support for some UNIQUE indexes on hash- or range-partitioned tables + + * Deprecates \stage in favor of using COPY for append-partition tables + + * Deprecates copy_to_distributed_table in favor of first-class COPY support + + * Fixes build problems when using non-packaged PostgreSQL installs + + * Fixes bug that sometimes skipped pruning when partitioned by VARCHAR column + + * Fixes bug impeding use of user functions in repartitioned subqueries + + * Fixes bug involving queries with equality comparisons of boolean types + + * Fixes crash that prevented use alongside pg_stat_statements + + * Fixes crash arising from SELECT queries that lack a target list + + * Improves warning and error messages + + -- Jason Petersen Tue, 17 May 2016 16:55:02 +0000 + +citus (5.1.0~rc.2-1) testing; urgency=low + + * Fix EXPLAIN output when FORMAT JSON in use + + -- Jason Petersen Mon, 16 May 2016 11:08:09 +0000 + +citus (5.1.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.1. + + -- Jason Petersen Wed, 04 May 2016 19:26:23 +0000 + +citus (5.0.1-1) stable; urgency=low + + * Fixes issues on 32-bit systems + + -- Jason Petersen Fri, 15 Apr 2016 19:17:35 +0000 + +citus (5.0.0-1) stable; urgency=low + + * Initial release + + -- Jason Petersen Thu, 24 Mar 2016 10:12:52 -0400 diff --git a/python/tests/files/pkgvars b/python/tests/files/pkgvars new file mode 100644 index 00000000..df2ecae5 --- /dev/null +++ b/python/tests/files/pkgvars @@ -0,0 +1,6 @@ +pkgname=citus +pkgdesc='Citus (Open-Source)' +pkglatest=10.0.2-1 +releasepg=11,12,13 +nightlypg=12,13 +versioning=fancy \ No newline at end of file diff --git a/python/tests/files/verify/debian_changelog_with_10.0.3.txt b/python/tests/files/verify/debian_changelog_with_10.0.3.txt new file mode 100644 index 00000000..469199fc --- /dev/null +++ b/python/tests/files/verify/debian_changelog_with_10.0.3.txt @@ -0,0 +1,1802 @@ +citus (10.0.3.citus-1) stable; urgency=low + + * Prevents infinite recursion for queries that involve `UNION ALL` + below `JOIN` + + * Fixes a crash in queries with a modifying `CTE` and a `SELECT` + without `FROM` + + * Fixes upgrade and downgrade paths for `citus_update_table_statistics` + + * Fixes a bug that causes `SELECT` queries to use 2PC unnecessarily + + * Fixes a bug that might cause self-deadlocks with + `CREATE INDEX` / `REINDEX CONCURRENTLY` commands + + * Adds `citus.max_cached_connection_lifetime` GUC to set maximum connection + lifetime + + * Adds `citus.remote_copy_flush_threshold` GUC that controls + per-shard memory usages by `COPY` + + * Adds `citus_get_active_worker_nodes` UDF to deprecate + `master_get_active_worker_nodes` + + * Skips 2PC for readonly connections in a transaction + + * Makes sure that local execution starts coordinated transaction + + * Removes open temporary file warning when cancelling a query with + an open tuple store + + * Relaxes the locks when adding an existing node + + -- Gurkan Indibay Thu, 18 Mar 2021 01:40:08 +0000 + +citus (10.0.2.citus-1) stable; urgency=low + + * Adds a configure flag to enforce security + + * Fixes a bug due to cross join without target list + + * Fixes a bug with UNION ALL on PG 13 + + * Fixes a compatibility issue with pg_audit in utility calls + + * Fixes insert query with CTEs/sublinks/subqueries etc + + * Grants SELECT permission on citus_tables view to public + + * Grants SELECT permission on columnar metadata tables to public + + * Improves citus_update_table_statistics and provides distributed deadlock + detection + + * Preserves colocation with procedures in alter_distributed_table + + * Prevents using alter_columnar_table_set and alter_columnar_table_reset + on a columnar table not owned by the user + + * Removes limits around long table names + + -- Gurkan Indibay Thu, 4 Mar 2021 2:46:54 +0000 + +citus (9.5.2.citus-1) stable; urgency=low + + * Fixes distributed deadlock detection being blocked by metadata sync + + * Prevents segfaults when SAVEPOINT handling cannot recover from connection + failures + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Wed, 27 Jan 2021 11:25:38 +0000 + +citus (9.4.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause router queries with local tables to be pushed + down + + * Fixes a segfault in connection management due to invalid connection hash + entries + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Tue, 5 Jan 2021 14:58:56 +0000 + +citus (9.5.1.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that could cause excessive memory consumption when a partition is + created + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + * Fixes a segfault in connection management due to invalid connection hash + entries + + -- Onur Tirtir Wed, 2 Dec 2020 14:28:44 +0000 + +citus (9.4.3.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + -- Onur Tirtir Tue, 24 Nov 2020 11:17:57 +0000 + +citus (9.5.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 13 + + * Removes the task-tracker executor + + * Introduces citus local tables + + * Introduces undistribute_table UDF to convert tables back to postgres tables + + * Adds support for EXPLAIN (ANALYZE) EXECUTE and EXPLAIN EXECUTE + + * Adds support for EXPLAIN (ANALYZE, WAL) for PG13 + + * Sorts the output of EXPLAIN (ANALYZE) by execution duration. + + * Adds support for CREATE TABLE ... USING table_access_method + + * Adds support for WITH TIES option in SELECT and INSERT SELECT queries + + * Avoids taking multi-shard locks on workers + + * Enforces citus.max_shared_pool_size config in COPY queries + + * Enables custom aggregates with multiple parameters to be executed on workers + + * Enforces citus.max_intermediate_result_size in local execution + + * Improves cost estimation of INSERT SELECT plans + + * Introduces delegation of procedures that read from reference tables + + * Prevents pull-push execution for simple pushdownable subqueries + + * Improves error message when creating a foreign key to a local table + + * Makes citus_prepare_pg_upgrade idempotent by dropping transition tables + + * Disallows ON TRUE outer joins with reference & distributed tables when + reference table is outer relation to avoid incorrect results + + * Disallows field indirection in INSERT/UPDATE queries to avoid incorrect + results + + * Disallows volatile functions in UPDATE subqueries to avoid incorrect results + + * Fixes CREATE INDEX CONCURRENTLY crash with local execution + + * Fixes citus_finish_pg_upgrade to drop all backup tables + + * Fixes a bug that cause failures when RECURSIVE VIEW joined reference table + + * Fixes DROP SEQUENCE failures when metadata syncing is enabled + + * Fixes a bug that caused CREATE TABLE with CHECK constraint to fail + + * Fixes a bug that could cause VACUUM to deadlock + + * Fixes master_update_node failure when no background worker slots are available + + * Fixes a bug that caused replica identity to not be propagated on shard repair + + * Fixes a bug that could cause crashes after connection timeouts + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug that could cause deadlocks on CREATE INDEX + + * Fixes a bug with genetic query optimization in outer joins + + * Fixes a crash when aggregating empty tables + + * Fixes a crash with inserting domain constrained composite types + + * Fixes a crash with multi-row & router INSERT's in local execution + + * Fixes a possibility of doing temporary file cleanup more than once + + * Fixes incorrect setting of join related fields + + * Fixes memory issues around deparsing index commands + + * Fixes reference table access tracking for sequential execution + + * Fixes removal of a single node with only reference tables + + * Fixes sending commands to coordinator when it is added as a worker + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlock + + -- Onur Tirtir Wed, 11 Nov 2020 15:00:27 +0000 + +citus (9.4.2.citus-1) stable; urgency=low + + * Fixes a bug that could lead to multiple maintenance daemons + + * Fixes an issue preventing views in reference table modifications + + -- Onur Tirtir Thu, 22 Oct 2020 8:53:44 +0000 + +citus (9.4.1.citus-1) stable; urgency=low + + * Fixes EXPLAIN ANALYZE output truncation + + * Fixes a deadlock during transaction recovery + + -- Onur Tirtir Wed, 30 Sep 2020 9:33:46 +0000 + +citus (9.4.0.citus-1) stable; urgency=low + + * Improves COPY by honoring max_adaptive_executor_pool_size config + + * Adds support for insert into local table select from distributed table + + * Adds support to partially push down tdigest aggregates + + * Adds support for receiving binary encoded results from workers using + citus.enable_binary_protocol + + * Enables joins between local tables and CTEs + + * Adds showing query text in EXPLAIN output when explain verbose is true + + * Adds support for showing CTE statistics in EXPLAIN ANALYZE + + * Adds support for showing amount of data received in EXPLAIN ANALYZE + + * Introduces downgrade paths in migration scripts + + * Avoids returning incorrect results when changing roles in a transaction + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes a bug that occurs when coordinator is added as a worker node + + * Fixes a crash because of overflow in partition id with certain compile flags + + * Fixes a crash that may happen if no worker nodes are added + + * Fixes a crash that occurs when inserting implicitly coerced constants + + * Fixes a crash when aggregating empty tables + + * Fixes a memory leak in subtransaction memory handling + + * Fixes crash when using rollback to savepoint after cancellation of DML + + * Fixes deparsing for queries with anonymous column references + + * Fixes distribution of composite types failing to include typemods + + * Fixes explain analyze on adaptive executor repartitions + + * Fixes possible error throwing in abort handle + + * Fixes segfault when evaluating func calls with default params on coordinator + + * Fixes several EXPLAIN ANALYZE issues + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlocks + + * Reports correct INSERT/SELECT method in EXPLAIN + + * Disallows triggers on citus tables + + -- Onur Tirtir Tue, 28 Jul 2020 13:22:31 +0000 + +citus (9.3.5.citus-1) stable; urgency=low + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a crash because of overflow in partition id with certain compile flags + + -- Onur Tirtir Mon, 27 Jul 2020 7:28:18 +0000 + +citus (9.3.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes deparsing for queries with anonymous column references + + -- Onur Tirtir Wed, 22 Jul 2020 9:00:01 +0000 + +citus (9.3.3.citus-1) stable; urgency=low + + * Fixes a memory leak in subtransaction memory handling + + -- Onur Tirtir Mon, 13 Jul 2020 8:47:40 +0000 + +citus (9.3.0.citus-1) stable; urgency=low + + * Adds max_shared_pool_size to control number of connections across sessions + + * Adds support for window functions on coordinator + + * Improves shard pruning logic to understand OR-conditions + + * Prevents using an extra connection for intermediate result multi-casts + + * Adds propagation of ALTER ROLE .. SET statements + + * Adds update_distributed_table_colocation UDF to update colocation of tables + + * Introduces a UDF to truncate local data after distributing a table + + * Adds support for creating temp schemas in parallel + + * Adds support for evaluation of nextval in the target list on coordinator + + * Adds support for local execution of COPY/TRUNCATE/DROP/DDL commands + + * Adds support for local execution of shard creation + + * Uses local execution in a transaction block + + * Adds support for querying distributed table sizes concurrently + + * Allows master_copy_shard_placement to replicate placements to new nodes + + * Allows table type to be used in target list + + * Avoids having multiple maintenance daemons active for a single database + + * Defers reference table replication to shard creation time + + * Enables joins between local tables and reference tables in transaction blocks + + * Ignores pruned target list entries in coordinator plan + + * Improves SIGTERM handling of maintenance daemon + + * Increases the default of citus.node_connection_timeout to 30 seconds + + * Fixes a bug that occurs when creating remote tasks in local execution + + * Fixes a bug that causes some DML queries containing aggregates to fail + + * Fixes a bug that could cause failures in queries with subqueries or CTEs + + * Fixes a bug that may cause some connection failures to throw errors + + * Fixes a bug which caused queries with SRFs and function evalution to fail + + * Fixes a bug with generated columns when executing COPY dist_table TO file + + * Fixes a crash when using non-constant limit clauses + + * Fixes a failure when composite types used in prepared statements + + * Fixes a possible segfault when dropping dist. table in a transaction block + + * Fixes a possible segfault when non-pushdownable aggs are solely used in HAVING + + * Fixes a segfault when executing queries using GROUPING + + * Fixes an error when using LEFT JOIN with GROUP BY on primary key + + * Fixes an issue with distributing tables having generated cols not at the end + + * Fixes automatic SSL permission issue when using "initdb --allow-group-access" + + * Fixes errors which could occur when subqueries are parameters to aggregates + + * Fixes possible issues by invalidating the plan cache in master_update_node + + * Fixes timing issues which could be caused by changing system clock + + -- Onur Tirtir Thu, 7 May 2020 15:11:25 +0000 + +citus (9.2.4.citus-1) stable; urgency=low + + * Fixes a release problem in 9.2.3 + + -- Onur Tirtir Tue, 31 Mar 2020 08:06:59 +0000 + +citus (9.2.3.citus-1) stable; urgency=low + + * Do not use C functions that have been banned by Microsoft + + * Fixes a bug that causes wrong results with complex outer joins + + * Fixes issues found using static analysis + + * Fixes left join shard pruning in pushdown planner + + * Fixes possibility for segmentation fault in internal aggregate functions + + * Fixes possible segfault when non pushdownable aggregates are used in HAVING + + * Improves correctness of planning subqueries in HAVING + + * Prevents using old connections for security if citus.node_conninfo changed + + * Uses Microsoft approved cipher string for default TLS setup + + -- Onur Tirtir Thu, 26 Mar 2020 8:22:48 +0000 + +citus (9.0.2.citus-1) stable; urgency=low + + * Fixes build errors on EL/OL 6 based distros + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Disallows distributed function creation when replication_model is `statement` + + -- Onur Tirtir Fri, 6 Mar 2020 14:10:16 +0000 + +citus (9.2.2.citus-1) stable; urgency=low + + * Fixes a bug that caused some prepared stmts with function calls to fail + + * Fixes a bug that caused some prepared stmts with composite types to fail + + * Fixes a bug that caused missing subplan results in workers + + * Improves performance of re-partition joins + + -- Onur Tirtir Fri, 6 Mar 2020 07:14:20 +0000 + +citus (9.2.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes if distribution key is NULL + + -- Onur Tirtir Fri, 14 Feb 2020 11:51:09 +0000 + +citus (9.2.0.citus-1) stable; urgency=low + + * Adds support for INSERT...SELECT queries with re-partitioning + + * Adds citus.coordinator_aggregation_strategy to support more aggregates + + * Adds caching of local plans on shards for Citus MX + + * Adds compatibility support for dist. object infrastructure from old versions + + * Adds defering shard-pruning for fast-path router queries to execution + + * Adds propagation of GRANT ... ON SCHEMA queries + + * Adds support for CTE pushdown via CTE inlining in distributed planning + + * Adds support for ALTER TABLE ... SET SCHEMA propagation. + + * Adds support for DROP ROUTINE & ALTER ROUTINE commands + + * Adds support for any inner join on a reference table + + * Changes citus.log_remote_commands level to NOTICE + + * Disallows marking ref. table shards unhealthy in the presence of savepoints + + * Disallows placing new shards with shards in TO_DELETE state + + * Enables local execution of queries that do not need any data access + + * Fixes Makefile trying to cleanup PG directory during install + + * Fixes a bug causing errors when planning a query with multiple subqueries + + * Fixes a possible deadlock that could happen during shard moves + + * Fixes a problem when adding a new node due to tables referenced in func body + + * Fixes an issue that could cause joins with reference tables to be slow + + * Fixes cached metadata for shard is inconsistent issue + + * Fixes inserting multiple composite types as partition key in VALUES + + * Fixes unnecessary repartition on joins with more than 4 tables + + * Prevents wrong results for replicated partitioned tables after failure + + * Restricts LIMIT approximation for non-commutative aggregates + + -- Onur Tirtir Wed, 10 Feb 2020 8:48:00 +0000 + +citus (9.1.1.citus-1) stable; urgency=low + + * Fixes a bug causing SQL-executing UDFs to crash when passing in DDL + + * Fixes a bug that caused column_to_column_name to crash for invalid input + + * Fixes a bug that caused inserts into local tables w/ dist. subqueries to crash + + * Fixes a bug that caused some noop DML statements to fail + + * Fixes a bug that prevents dropping reference table columns + + * Fixes a crash in IN (.., NULL) queries + + * Fixes a crash when calling a distributed function from PL/pgSQL + + * Fixes an issue that caused CTEs to sometimes leak connections + + * Fixes strange errors in DML with unreachable sublinks + + * Prevents statements in SQL functions to run outside of a transaction + + -- Onur Tirtir Wed, 18 Dec 2019 14:32:42 +0000 + +citus (9.1.0.citus-1) stable; urgency=low + + * Adds extensions to distributed object propagation infrastructure + + * Adds support for ALTER ROLE propagation + + * Adds support for aggregates in create_distributed_function + + * Adds support for expressions in reference joins + + * Adds support for returning RECORD in multi-shard queries + + * Adds support for simple IN subqueries on unique cols in repartition joins + + * Adds support for subqueries in HAVING clauses + + * Automatically distributes unary aggs w/ combinefunc and non-internal stype + + * Disallows distributed func creation when replication_model is 'statement' + + * Drops support for deprecated real-time and router executors + + * Fixes a bug in local execution that could cause missing rows in RETURNING + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Fixes a bug that caused other CREATE EXTENSION commands to take longer + + * Fixes a bug that prevented REFRESH MATERIALIZED VIEW + + * Fixes a bug when view is used in modify statements + + * Fixes a memory leak in adaptive executor when query returns many columns + + * Fixes underflow init of default values in worker extended op node creation + + * Fixes potential segfault in standard_planner inlining functions + + * Fixes an issue that caused failures in RHEL 6 builds + + * Fixes queries with repartition joins and group by unique column + + * Improves CTE/Subquery performance by pruning intermediate rslt broadcasting + + * Removes citus.worker_list_file GUC + + * Revokes usage from the citus schema from public + + -- Onur Tirtir Thu, 28 Nov 2019 15:11:05 +0000 + +citus (9.0.1.citus-1) stable; urgency=low + + * Fixes a memory leak in the executor + + * Revokes usage from the citus schema from public + + -- Hanefi Onaldi Wed, 30 Oct 2019 8:53:22 +0000 + +citus (9.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 12 + + * Adds UDFs to help with PostgreSQL upgrades + + * Distributes types to worker nodes + + * Introduces create_distributed_function UDF + + * Introduces local query execution for Citus MX + + * Implements infrastructure for routing CALL to MX workers + + * Implements infrastructure for routing SELECT function() to MX workers + + * Adds support for foreign key constraints between reference tables + + * Adds a feature flag to turn off CREATE TYPE propagation + + * Adds option citus.single_shard_commit_protocol + + * Adds support for EXPLAIN SUMMARY + + * Adds support for GENERATE ALWAYS AS STORED + + * Adds support for serial and smallserial in MX mode + + * Adds support for anon composite types on the target list in router queries + + * Avoids race condition between create_reference_table & master_add_node + + * Fixes a bug in schemas of distributed sequence definitions + + * Fixes a bug that caused run_command_on_colocated_placements to fail + + * Fixes a bug that leads to various issues when a connection is lost + + * Fixes a schema leak on CREATE INDEX statement + + * Fixes assert failure in bare SELECT FROM reference table FOR UPDATE in MX + + * Makes master_update_node MX compatible + + * Prevents pg_dist_colocation from multiple records for reference tables + + * Prevents segfault in worker_partition_protocol edgecase + + * Propagates ALTER FUNCTION statements for distributed functions + + * Propagates CREATE OR REPLACE FUNCTION for distributed functions + + * Propagates REINDEX on tables & indexes + + * Provides a GUC to turn of the new dependency propagation functionality + + * Uses 2PC in adaptive executor when dealing with replication factors above 1 + + -- Hanefi Onaldi Tue, 15 Oct 2019 16:54:50 +0000 + +citus (8.3.2.citus-1) stable; urgency=low + + * Fixes performance issues by skipping unnecessary relation access recordings + + -- Hanefi Onaldi Fri, 9 Aug 2019 11:15:57 +0000 + +citus (8.3.1.citus-1) stable; urgency=low + + * Improves Adaptive Executor performance + + -- Hanefi Onaldi Mon, 29 Jul 2019 10:25:50 +0000 + +citus (8.3.0.citus-1) stable; urgency=low + + * Adds a new distributed executor: Adaptive Executor + + * citus.enable_statistics_collection defaults to off (opt-in) + + * Adds support for CTEs in router planner for modification queries + + * Adds support for propagating SET LOCAL at xact start + + * Adds option to force master_update_node during failover + + * Deprecates master_modify_multiple_shards + + * Improves round robin logic on router queries + + * Creates all distributed schemas as superuser on a separate connection + + * Makes COPY adapt to connection use behaviour of previous commands + + * Replaces SESSION_LIFESPAN with configurable no. of connections at xact end + + * Propagates ALTER FOREIGN TABLE commands to workers + + * Don't schedule tasks on inactive nodes + + * Makes DROP/VALIDATE CONSTRAINT tolerant of ambiguous shard extension + + * Fixes an issue with subquery map merge jobs as non-root + + * Fixes null pointers caused by partial initialization of ConnParamsHashEntry + + * Fixes errors caused by joins with shadowed aliases + + * Fixes a regression in outer joining subqueries introduced in 8.2.0 + + * Fixes a crash that can occur under high memory load + + * Fixes a bug that selects wrong worker when using round-robin assignment + + * Fixes savepoint rollback after multi-shard modify/copy failure + + * Fixes bad foreign constraint name search + + * Fixes a bug that prevents stack size to be adjusted + + -- Hanefi Onaldi Wed, 10 Jul 2019 15:19:02 +0000 + +citus (8.2.2.citus-1) stable; urgency=low + + * Fixes a bug in outer joins wrapped in subqueries + + -- Burak Velioglu Wed, 12 Jun 2019 8:45:08 +0000 + +citus (8.2.1.citus-1) stable; urgency=low + + * Fixes a bug that prevents stack size to be adjusted + + -- Burak Velioglu Wed, 3 Apr 2019 20:56:47 +0000 + +citus (8.1.2.citus-1) stable; urgency=low + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Fixes a bug that prevents stack size to be adjusted + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + -- Burak Velioglu Wed, 3 Apr 2019 20:34:46 +0000 + +citus (8.2.0.citus-1) stable; urgency=low + + * Removes support and code for PostgreSQL 9.6 + + * Enable more outer joins with reference tables + + * Execute CREATE INDEX CONCURRENTLY in parallel + + * Treat functions as transaction blocks + + * Add support for column aliases on join clauses + + * Skip standard_planner() for trivial queries + + * Added support for function calls in joins + + * Round-robin task assignment policy relies on local transaction id + + * Relax subquery union pushdown restrictions for reference tables + + * Speed-up run_command_on_shards() + + * Address some memory issues in connection config + + * Restrict visibility of get_*_active_transactions functions to pg_monitor + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Queries with only intermediate results do not rely on task assignment policy + + * Finish connection establishment in parallel for multiple connections + + * Fixes a bug related to pruning shards using a coerced value + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + * Fixes a bug that could lead to infinite recursion during recursive planning + + * Fixes a bug that could prevent planning full outer joins with using clause + + * Fixes a bug that could lead to memory leak on citus_relation_size + + * Fixes a problem that could cause segmentation fault with recursive planning + + * Switch CI solution to CircleCI + + -- Burak Velioglu Fri, 29 Mar 2019 07:36:09 +0000 + +citus (8.0.3.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Wed, 9 Jan 2019 9:50:07 +0000 + +citus (8.1.1.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Mon, 7 Jan 2019 16:26:13 +0000 + +citus (8.1.0.citus-1) stable; urgency=low + + * Turns on ssl by default for new installations of citus + + * Restricts SSL Ciphers to TLS1.2 and above + + * Adds support for INSERT INTO SELECT..ON CONFLICT/RETURNING via coordinator + + * Adds support for round-robin task assignment for queries to reference tables + + * Adds support for SQL tasks using worker_execute_sql_task UDF with task-tracker + + * Adds support for VALIDATE CONSTRAINT queries + + * Adds support for disabling hash aggregate with HLL + + * Adds user ID suffix to intermediate files generated by task-tracker + + * Only allow transmit from pgsql_job_cache directory + + * Disallows GROUPING SET clauses in subqueries + + * Removes restriction on user-defined group ID in node addition functions + + * Relaxes multi-shard modify locks when enable_deadlock_prevention is disabled + + * Improves security in task-tracker protocol + + * Improves permission checks in internal DROP TABLE functions + + * Improves permission checks in cluster management functions + + * Cleans up UDFs and fixes permission checks + + * Fixes crashes caused by stack size increase under high memory load + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Tue, 18 Dec 2018 15:12:45 +0000 + +citus (8.0.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Thu, 13 Dec 2018 13:56:44 +0000 + +citus (7.5.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Wed, 12 Dec 2018 11:45:24 +0000 + +citus (8.0.1.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 11:38:47 +0000 + +citus (7.5.3.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 10:52:20 +0000 + +citus (7.5.2.citus-1) stable; urgency=low + + * Fixes inconsistent metadata error when shard metadata caching get interrupted + + * Fixes a bug that could cause memory leak + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Wed, 14 Nov 2018 20:42:16 +0000 + +citus (8.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 11 + + * Adds support for applying DML operations on reference tables from MX nodes + + * Adds distributed locking to truncated MX tables + + * Adds support for running TRUNCATE command from MX worker nodes + + * Adds views to provide insight about the distributed transactions + + * Adds support for TABLESAMPLE in router queries + + * Adds support for INCLUDE option in index creation + + * Adds option to allow simple DML commands from hot standby + + * Adds support for partitioned tables with replication factor > 1 + + * Prevents a deadlock on concurrent DROP TABLE and SELECT on Citus MX + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes a bug in MX mode, calling DROP SCHEMA with existing partitioned table + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + * Fixes a bug preventing rollback in CREATE PROCEDURE + + * Fixes a bug on not being able to drop index on a partitioned table + + * Fixes a bug on TRUNCATE when there is a foreign key to a reference table + + * Fixes a performance issue in prepared INSERT..SELECT + + * Fixes a bug which causes errors on DROP DATABASE IF EXISTS + + * Fixes a bug to remove intermediate result directory in pull-push execution + + * Improves query pushdown planning performance + + * Evaluate functions anywhere in query + + -- Burak Velioglu Fri, 02 Nov 2018 08:06:42 +0000 + +citus (7.5.1.citus-1) stable; urgency=low + + * Improves query pushdown planning performance + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + -- Burak Velioglu Wed, 29 Aug 2018 08:06:42 +0000 + +citus (7.4.2.citus-1) stable; urgency=low + + * Fixes a segfault in real-time executor during online shard move + + -- Mehmet Furkan Sahin Fri, 27 Jul 2018 13:42:27 +0000 + +citus (7.5.0.citus-1) stable; urgency=low + + * Adds foreign key support from hash distributed to reference tables + + * Adds SELECT ... FOR UPDATE support for router plannable queries + + * Adds support for non-partition columns in count distinct + + * Fixes a segfault in real-time executor during online shard move + + * Fixes ALTER TABLE ADD COLUMN constraint check + + * Fixes a bug where INSERT ... SELECT allows one to update dist. column + + * Allows DDL commands to be sequentialized via citus.multi_shard_modify_mode + + * Adds support for topn_union_agg and topn_add_agg across shards + + * Adds support for hll_union_agg and hll_add_agg across shards + + * Fixes a bug that might cause shards to have a wrong owner + + * Adds select_opens_transaction_block GUC + + * Adds utils to implement DDLs for policies in future + + * Makes intermediate results to use separate connections + + * Adds a node_conninfo GUC to set outgoing connection settings + + -- Mehmet Furkan Sahin Wed, 25 Jul 2018 9:32:24 +0000 + +citus (6.2.6.citus-1) stable; urgency=low + + * Adds support for respecting enable_hashagg in the master planner + + -- Burak Velioglu Fri, 06 Jul 2018 13:30:08 +0000 + +citus (7.4.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause txns to incorrectly proceed after failure + + * Fixes a bug on INSERT ... SELECT queries in prepared statements + + -- Burak Velioglu Wed, 20 Jun 2018 12:25:30 +0000 + +citus (7.2.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause SELECTs to crash during a rebalance + + -- Burak Velioglu Thu, 17 May 2018 11:51:56 +0000 + +citus (7.4.0.citus-1) stable; urgency=low + + * Adds support for non-pushdownable subqueries and CTEs in UPDATE/DELETE + + * Adds support for pushdownable subqueries and joins in UPDATE/DELETE + + * Adds faster shard pruning for subqueries + + * Adds partitioning support to MX table + + * Adds support for (VACUUM | ANALYZE) VERBOSE + + * Adds support for multiple ANDs in HAVING for pushdown planner + + * Adds support for quotation needy schema names + + * Improves operator check time in physical planner for custom data types + + * Removes broadcast join logic + + * Deprecates large_table_shard_count and master_expire_table_cache() + + * Modifies master_update_node to write-lock shards hosted by node over update + + * DROP TABLE now drops shards as the currrent user instead of the superuser + + * Adds specialised error codes for connection failures + + * Improves error messages on connection failure + + * Fixes issue which prevented multiple citus_table_size calls per query + + * Tests are updated to use create_distributed_table + + -- Burak Velioglu Tue, 15 May 2018 13:01:17 +0000 + +citus (7.3.0.citus-1) stable; urgency=low + + * Adds support for non-colocated joins between subqueries + + * Adds support for window functions that can be pushed down to worker + + * Adds support for modifying CTEs + + * Adds recursive plan for WHERE clause subqueries with recurring FROM clause + + * Adds support for bool_ and bit_ aggregates + + * Adds support for Postgres jsonb and json aggregation functions + + * Adds support for respecting enable_hashagg in the master plan + + * Performance improvements to reduce distributed planning time + + * Fixes a bug on planner when aggregate is used in ORDER BY + + * Fixes a bug on planner when DISTINCT (ON) clause is used with GROUP BY + + * Fixes a planner bug with distinct and aggregate clauses + + * Fixes a bug that opened new connections on each table size function call + + * Fixes a bug canceling backends not involved in distributed deadlocks + + * Fixes count distinct bug on column expressions when used with subqueries + + * Improves error handling on worker node failures + + * Improves error messages for INSERT queries that have subqueries + + -- Burak Velioglu Thu, 15 Mar 2018 14:16:10 +0000 + +citus (7.2.1.citus-1) stable; urgency=low + + * Fixes count distinct bug on column expressions when used with subqueries + + * Adds support for respecting enable_hashagg in the master plan + + * Fixes a bug canceling backends not involved in distributed deadlocks + + -- Burak Velioglu Tue, 06 Feb 2018 14:46:07 +0000 + +citus (7.2.0.citus-1) stable; urgency=low + + * Adds support for CTEs + + * Adds support for subqueries that require merge step + + * Adds support for set operations (UNION, INTERSECT, ...) + + * Adds support for 2PC auto-recovery + + * Adds support for querying local tables in CTEs and subqueries + + * Adds support for more SQL coverage in subqueries for reference tables + + * Adds support for count(distinct) in queries with a subquery + + * Adds support for non-equijoins when there is already an equijoin + + * Adds support for real-time executor to run in transaction blocks + + * Adds infrastructure for storing intermediate distributed query results + + * Adds a new GUC named enable_repartition_joins for auto executor switch + + * Adds support for limiting the intermediate result size + + * Improves support for queries with unions containing filters + + * Improves support for queries with unions containing joins + + * Improves support for subqueries in the WHERE clause + + * Increases COPY throughput + + * Enables pushing down queries containing only recurring tuples and GROUP BY + + * Load-balance queries that read from 0 shards + + * Improves support for using functions in subqueries + + * Fixes a bug that causing real-time executor to crash during cancellation + + * Fixes a bug that causing real-time executor to get stuck on cancellation + + * Fixes a bug that could block modification queries unnecessarily + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Fixes a bug that could cause an assert failure with ANALYZE statements + + * Fixes a bug that would push down wrong set operations in subqueries + + * Fixes a bug that could cause a deadlock in create_distributed_table + + * Fixes a bug that could confuse user about ANALYZE usage + + * Fixes a bug causing false positive distributed deadlock detections + + * Relaxes the locking for DDL commands on partitioned tables + + * Relaxes the locking on COPY with replication + + * Logs more remote commands when citus.log_remote_commands is set + + -- Burak Velioglu Tue, 16 Jan 2018 14:34:20 +0000 + +citus (6.2.5.citus-1) stable; urgency=low + + * Fixes a bug that could crash the coordinator while reporting a remote error + + -- Burak Velioglu Thu, 11 Jan 2018 11:40:28 +0000 + +citus (7.1.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Increases COPY throughput + + -- Burak Velioglu Fri, 05 Jan 2018 09:00:07 +0000 + +citus (7.1.1.citus-1) stable; urgency=low + + * Fixes a bug preventing pushing down subqueries with reference tables + + * Fixes a bug that could create false positive distributed deadlocks + + * Fixes a bug that could prevent running concurrent COPY and multi-shard DDL + + * Fixes a bug that could mislead users about ANALYZE queries + + -- Burak Velioglu Tue, 05 Dec 2017 09:00:07 +0000 + +citus (7.1.0.citus-1) stable; urgency=low + + * Adds support for native queries with multi shard UPDATE/DELETE queries + + * Expands reference table support in subquery pushdown + + * Adds window function support for subqueries and INSERT ... SELECT queries + + * Adds support for COUNT(DISTINCT) [ON] queries on non-partition columns + + * Adds support for DISTINCT [ON] queries on non-partition columns + + * Introduces basic usage statistic collector + + * Adds support for setting replica identity while creating distributed tables + + * Adds support for ALTER TABLE ... REPLICA IDENTITY queries + + * Adds pushdown support for LIMIT and HAVING grouped by partition key + + * Adds support for INSERT ... SELECT queries via worker nodes on MX clusters + + * Adds support for adding primary key using already defined index + + * Adds replication parameter to shard copy functions + + * Changes shard_name UDF to omit public schema name + + * Adds master_move_node UDF to make changes on nodename/nodeport more easy + + * Fixes a bug that could cause casting error with INSERT ... SELECT queries + + * Fixes a bug that could prevent upgrading servers from Citus 6.1 + + * Fixes a bug that could prevent attaching partitions to a table in schema + + * Fixes a bug preventing adding nodes to clusters with reference tables + + * Fixes a bug that could cause a crash with INSERT ... SELECT queries + + * Fixes a bug that could prevent creating a partitoned table on Cloud + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Velioglu Wed, 15 Nov 2017 09:00:07 +0000 + +citus (7.0.3.citus-1) stable; urgency=low + + * Fixes several bugs that could cause crash + + * Fixes a bug that could cause deadlock while creating reference tables + + * Fixes a bug that could cause false-positives in deadlock detection + + * Fixes a bug that could cause 2PC recovery not to work from MX workers + + * Fixes a bug that could cause cache incohorency + + * Fixes a bug that could cause maintenance daemon to skip cache invalidations + + * Improves performance of transaction recovery by using correct index + + -- Burak Yucesoy Mon, 16 Oct 2017 11:52:07 +0000 + +citus (7.0.2.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 22:29:01 +0000 + +citus (6.2.4.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 21:31:35 +0000 + +citus (6.1.3.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 20:31:35 +0000 + +citus (7.0.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause memory leaks in INSERT ... SELECT queries + + * Fixes a bug that could cause incorrect execution of prepared statements + + * Fixes a bug that could cause excessive memory usage during COPY + + * Incorporates latest changes from core PostgreSQL code + + -- Burak Yucesoy Tue, 12 Sep 2017 17:53:50 +0000 + +citus (7.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 10 + + * Drops support for PostgreSQL 9.5 + + * Adds support for multi-row INSERT + + * Adds support for router UPDATE and DELETE queries with subqueries + + * Adds infrastructure for distributed deadlock detection + + * Deprecates enable_deadlock_prevention flag + + * Adds support for partitioned tables + + * Adds support for creating UNLOGGED tables + + * Adds support for SAVEPOINT + + * Adds UDF citus_create_restore_point for taking distributed snapshots + + * Adds support for evaluating non-pushable INSERT ... SELECT queries + + * Adds support for subquery pushdown on reference tables + + * Adds shard pruning support for IN and ANY + + * Adds support for UPDATE and DELETE commands that prune down to 0 shard + + * Enhances transaction support by relaxing some transaction restrictions + + * Fixes a bug causing crash if distributed table has no shards + + * Fixes a bug causing crash when removing inactive node + + * Fixes a bug causing failure during COPY on tables with dropped columns + + * Fixes a bug causing failure during DROP EXTENSION + + * Fixes a bug preventing executing VACUUM and INSERT concurrently + + * Fixes a bug in prepared INSERT statements containing an implicit cast + + * Fixes several issues related to statement cancellations and connections + + * Fixes several 2PC related issues + + * Removes an unnecessary dependency causing warning messages in pg_dump + + * Adds internal infrastructure for follower clusters + + * Adds internal infrastructure for progress tracking + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Mon, 28 Aug 2017 12:27:53 +0000 + +citus (6.2.3.citus-1) stable; urgency=low + + * Fixes a crash during execution of local CREATE INDEX CONCURRENTLY + + * Fixes a bug preventing usage of quoted column names in COPY + + * Fixes a bug in prepared INSERTs with implicit cast in partition column + + * Relaxes locks in VACUUM to ensure concurrent execution with INSERT + + -- Burak Yucesoy Thu, 13 Jul 2017 11:27:17 +0000 + +citus (6.2.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Burak Velioglu Wed, 07 Jun 2017 09:42:17 +0000 + +citus (6.1.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Jason Petersen Wed, 31 May 2017 16:14:11 +0000 + +citus (6.2.1.citus-1) stable; urgency=low + + * Relaxes version-check logic to avoid breaking non-distributed commands + + -- Jason Petersen Wed, 24 May 2017 22:36:07 +0000 + +citus (6.2.0.citus-1) stable; urgency=low + + * Increases SQL subquery coverage by pushing down more kinds of queries + + * Adds CustomScan API support to allow read-only transactions + + * Adds support for CREATE/DROP INDEX CONCURRENTLY + + * Adds support for ALTER TABLE ... ADD CONSTRAINT + + * Adds support for ALTER TABLE ... RENAME COLUMN + + * Adds support for DISABLE/ENABLE TRIGGER ALL + + * Adds support for expressions in the partition column in INSERTs + + * Adds support for query parameters in combination with function evaluation + + * Adds support for creating distributed tables from non-empty local tables + + * Adds UDFs to get size of distributed tables + + * Adds UDFs to add a new node without replicating reference tables + + * Adds checks to prevent running Citus binaries with wrong metadata tables + + * Improves shard pruning performance for range queries + + * Improves planner performance for joins involving co-located tables + + * Improves shard copy performance by creating indexes after copy + + * Improves task-tracker performance by batching several status checks + + * Enables router planner for queries on range partitioned table + + * Changes TRUNCATE to drop local data only if enable_ddl_propagation is off + + * Starts to execute DDL on coordinator before workers + + * Fixes a bug causing incorrectly reading invalidated cache + + * Fixes a bug related to creation of schemas in workers with incorrect owner + + * Fixes a bug related to concurrent run of shard drop functions + + * Fixes a bug related to EXPLAIN ANALYZE with DML queries + + * Fixes a bug related to SQL functions in FROM clause + + * Adds a GUC variable to report cross shard queries + + * Fixes a bug related to partition columns without native hash function + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Tue, 16 May 2017 16:05:22 +0000 + +citus (6.1.1.citus-1) stable; urgency=low + + * Fixes a crash caused by router executor use after connection timeouts + + * Fixes a crash caused by relation cache invalidation during COPY + + * Fixes bug related to DDL use within PL/pgSQL functions + + * Fixes a COPY bug related to types lacking binary output functions + + * Fixes a bug related to modifications with parameterized partition values + + * Fixes improper value interpolation in worker sequence generation + + * Guards shard pruning logic against zero-shard tables + + * Fixes possible NULL pointer dereference and buffer underflow, via PVS-Studio + + * Fixes a INSERT...SELECT bug that could push down non-partition column JOINs + + -- Metin Doslu Fri, 5 May 2017 17:42:00 +0000 + +citus (6.1.0.citus-1) stable; urgency=low + + * Implements reference tables, transactionally replicated to all nodes + + * Adds upgrade_to_reference_table UDF to upgrade pre-6.1 reference tables + + * Expands prepared statement support to nearly all statements + + * Adds support for creating VIEWs which reference distributed tables + + * Adds targeted VACUUM/ANALYZE support + + * Adds support for the FILTER clause in aggregate expressions + + * Adds support for function evaluation within INSERT INTO ... SELECT + + * Adds support for creating foreign key constraints with ALTER TABLE + + * Adds logic to choose router planner for all queries it supports + + * Enhances create_distributed_table with parameter for explicit colocation + + * Adds generally useful utility UDFs previously available as "Citus Tools" + + * Adds user-facing UDFs for locking shard resources and metadata + + * Refactors connection and transaction management for more consistency + + * Enhances COPY with fully transactional semantics + + * Improves support for cancellation for a number of queries and commands + + * Adds column_to_column_name UDF to help users understand partkey values + + * Adds master_disable_node UDF for temporarily disabling nodes + + * Adds proper MX ("masterless") metadata propagation logic + + * Adds start_metadata_sync_to_node UDF to propagate metadata changes to nodes + + * Enhances SERIAL compatibility with MX tables + + * Adds an node_connection_timeout parameter to set node connection timeouts + + * Adds enable_deadlock_prevention setting to permit multi-node transactions + + * Adds a replication_model setting to specify replication of new tables + + * Changes the shard_replication_factor setting's default value to one + + * Adds code to automatically set max_prepared_transactions if not configured + + * Accelerates lookup of colocated shard placements + + * Fixes a bug affecting INSERT INTO ... SELECT queries using constant values + + * Fixes a bug by ensuring COPY does not mark placements inactive + + * Fixes a bug affecting reads from pg_dist_shard_placement table + + * Fixes a crash triggered by creating a foreign key without a column + + * Fixes a crash related to accessing catalog tables after aborted transaction + + * Fixes a bug affecting JOIN queries requiring repartitions + + * Fixes a bug affecting node insertions to pg_dist_node table + + * Fixes a crash triggered by queries with modifying common table expressions + + * Fixes a bug affecting workloads with concurrent shard appends and deletions + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Thu, 9 Feb 2017 16:17:41 +0000 + +citus (6.0.1.citus-3) stable; urgency=low + + * First build using new versioning practices + + -- Jason Petersen Wed, 8 Feb 2017 23:19:46 +0000 + +citus (6.0.1.citus-2) stable; urgency=low + + * Transitional package to guide users to new package name + + -- Jason Petersen Mon, 6 Feb 2017 16:33:44 +0000 + +citus (6.0.1.citus-1) stable; urgency=low + + * Fixes a bug causing failures during pg_upgrade + + * Fixes a bug preventing DML queries during colocated table creation + + * Fixes a bug that caused NULL parameters to be incorrectly passed as text + + -- Burak Yucesoy Wed, 30 Nov 2016 15:27:38 +0000 + +citus (6.0.0.citus-1) stable; urgency=low + + * Adds compatibility with PostgreSQL 9.6, now the recommended version + + * Removes the pg_worker_list.conf file in favor of a pg_dist_node table + + * Adds master_add_node and master_add_node UDFs to manage membership + + * Removes the \stage command and corresponding csql binary in favor of COPY + + * Removes copy_to_distributed_table in favor of first-class COPY support + + * Adds support for multiple DDL statements within a transaction + + * Adds support for certain foreign key constraints + + * Adds support for parallel INSERT INTO ... SELECT against colocated tables + + * Adds support for the TRUNCATE command + + * Adds support for HAVING clauses in SELECT queries + + * Adds support for EXCLUDE constraints which include the partition column + + * Adds support for system columns in queries (tableoid, ctid, etc.) + + * Adds support for relation name extension within INDEX definitions + + * Adds support for no-op UPDATEs of the partition column + + * Adds several general-purpose utility UDFs to aid in Citus maintenance + + * Adds master_expire_table_cache UDF to forcibly expire cached shards + + * Parallelizes the processing of DDL commands which affect distributed tables + + * Adds support for repartition jobs using composite or custom types + + * Enhances object name extension to handle long names and large shard counts + + * Parallelizes the master_modify_multiple_shards UDF + + * Changes distributed table creation to error if target table is not empty + + * Changes the pg_dist_shard.logicalrelid column from an oid to regclass + + * Adds a placementid column to pg_dist_shard_placement, replacing Oid use + + * Removes the pg_dist_shard.shardalias distribution metadata column + + * Adds pg_dist_partition.repmodel to track tables using streaming replication + + * Adds internal infrastructure to take snapshots of distribution metadata + + * Addresses the need to invalidate prepared statements on metadata changes + + * Adds a mark_tables_colocated UDF for denoting pre-6.0 manual colocation + + * Fixes a bug affecting prepared statement execution within PL/pgSQL + + * Fixes a bug affecting COPY commands using composite types + + * Fixes a bug that could cause crashes during EXPLAIN EXECUTE + + * Separates worker and master job temporary folders + + * Eliminates race condition between distributed modification and repair + + * Relaxes the requirement that shard repairs also repair colocated shards + + * Implements internal functions to track which tables' shards are colocated + + * Adds pg_dist_partition.colocationid to track colocation group membership + + * Extends shard copy and move operations to respect colocation settings + + * Adds pg_dist_local_group to prepare for future MX-related changes + + * Adds create_distributed_table to easily create shards and infer colocation + + -- Jason Petersen Tue, 8 Nov 2016 19:45:45 +0000 + +citus (5.2.2.citus-1) stable; urgency=low + + * Adds support for IF NOT EXISTS clause of CREATE INDEX command + + * Adds support for RETURN QUERY and FOR ... IN PL/pgSQL features + + * Extends the router planner to handle more queries + + * Changes COUNT of zero-row sets to return 0 rather than an empty result + + * Reduces the minimum permitted task_tracker_delay to a single millisecond + + * Fixes a bug that caused crashes during joins with a WHERE false clause + + * Fixes a bug triggered by unique violation errors raised in long txns + + * Fixes a bug resulting in multiple registration of transaction callbacks + + * Fixes a bug which could result in stale reads of distribution metadata + + * Fixes a bug preventing distributed modifications in some PL/pgSQL functions + + * Fixes some code paths that could hypothetically read uninitialized memory + + * Lowers log level of "waiting for activity" messages + + -- Jason Petersen Tue, 8 Nov 2016 18:43:37 +0000 + +citus (5.2.1.citus-1) stable; urgency=low + + * Fixes subquery pushdown to properly extract outer join qualifiers + + * Addresses possible memory leak during multi-shard transactions + + -- Jason Petersen Tue, 6 Sep 2016 20:47:15 +0000 + +citus (5.2.0.citus-1) stable; urgency=low + + * Drops support for PostgreSQL 9.4; PostgreSQL 9.5 is required + + * Adds schema support for tables, named objects (types, operators, etc.) + + * Evaluates non-immutable functions on master in all modification commands + + * Adds support for SERIAL types in non-partition columns + + * Adds support for RETURNING clause in INSERT, UPDATE, and DELETE commands + + * Adds support for multi-statement transactions using a fixed set of nodes + + * Full SQL support for SELECT queries which can be executed on single worker + + * Adds option to perform DDL changes using prepared transactions (2PC) + + * Adds an enable_ddl_propagation parameter to control DDL propagation + + * Accelerates shard pruning during merges + + * Adds master_modify_multiple_shards UDF to modify many shards at once + + * Adds COPY support for arrays of user-defined types + + * Now supports parameterized prepared statements for certain use cases + + * Extends LIMIT/OFFSET support to all executor types + + * Constraint violations now fail fast rather than hitting all placements + + * Makes master_create_empty_shard aware of shard placement policy + + * Reduces unnecessary sleep during queries processed by real-time executor + + * Improves task tracker executor's task cleanup logic + + * Relaxes restrictions on cancellation of DDL commands + + * Removes ONLY keyword from worker SELECT queries + + * Error message improvements and standardization + + * Moves master_update_shard_statistics function to pg_catalog schema + + * Fixes a bug where hash-partitioned anti-joins could return bad results + + * Now sets storage type correctly for foreign table-backed shards + + * Fixes master_update_shard_statistics issue with hash-partitioned tables + + * Fixes an issue related to extending table names that require escaping + + * Reduces risk of row counter overflows during modifications + + * Fixes a crash related to FILTER clause use in COUNT DISTINCT subqueries + + * Fixes crashes related to partition columns with high attribute numbers + + * Fixes certain subquery and join crashes + + * Detects flex for build even if PostgreSQL was built without it + + * Fixes assert-enabled crash when all_modifications_commutative is true + + -- Jason Petersen Wed, 17 Aug 2016 10:23:21 +0000 + +citus (5.2.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.2. + + -- Jason Petersen Mon, 01 Aug 2016 17:01:05 +0000 + +citus (5.1.1-1) stable; urgency=low + + * Adds complex count distinct expression support in repartitioned subqueries + + * Improves task tracker job cleanup logic, addressing a memory leak + + * Fixes bug that generated incorrect results for LEFT JOIN queries + + * Improves compatibility with Debian's reproducible builds project + + * Fixes build issues on FreeBSD platforms + + -- Jason Petersen Fri, 17 Jun 2016 16:20:15 +0000 + +citus (5.1.0-1) stable; urgency=low + + * Adds distributed COPY to rapidly populate distributed tables + + * Adds support for using EXPLAIN on distributed queries + + * Recognizes and fast-paths single-shard SELECT statements automatically + + * Increases INSERT throughput via shard pruning optimizations + + * Improves planner performance for joins involving tables with many shards + + * Adds ability to pass columns as arguments to function calls in UPDATEs + + * Introduces transaction manager for use by multi-shard commands + + * Adds COUNT(DISTINCT ...) pushdown optimization for hash-partitioned tables + + * Adds support for some UNIQUE indexes on hash- or range-partitioned tables + + * Deprecates \stage in favor of using COPY for append-partition tables + + * Deprecates copy_to_distributed_table in favor of first-class COPY support + + * Fixes build problems when using non-packaged PostgreSQL installs + + * Fixes bug that sometimes skipped pruning when partitioned by VARCHAR column + + * Fixes bug impeding use of user functions in repartitioned subqueries + + * Fixes bug involving queries with equality comparisons of boolean types + + * Fixes crash that prevented use alongside pg_stat_statements + + * Fixes crash arising from SELECT queries that lack a target list + + * Improves warning and error messages + + -- Jason Petersen Tue, 17 May 2016 16:55:02 +0000 + +citus (5.1.0~rc.2-1) testing; urgency=low + + * Fix EXPLAIN output when FORMAT JSON in use + + -- Jason Petersen Mon, 16 May 2016 11:08:09 +0000 + +citus (5.1.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.1. + + -- Jason Petersen Wed, 04 May 2016 19:26:23 +0000 + +citus (5.0.1-1) stable; urgency=low + + * Fixes issues on 32-bit systems + + -- Jason Petersen Fri, 15 Apr 2016 19:17:35 +0000 + +citus (5.0.0-1) stable; urgency=low + + * Initial release + + -- Jason Petersen Thu, 24 Mar 2016 10:12:52 -0400 diff --git a/python/tests/files/verify/expected_changelog_10.0.3.txt b/python/tests/files/verify/expected_changelog_10.0.3.txt new file mode 100644 index 00000000..889d3414 --- /dev/null +++ b/python/tests/files/verify/expected_changelog_10.0.3.txt @@ -0,0 +1,33 @@ +### citus v10.0.3 (March 16, 2021) ### + +* Prevents infinite recursion for queries that involve `UNION ALL` + below `JOIN` + +* Fixes a crash in queries with a modifying `CTE` and a `SELECT` + without `FROM` + +* Fixes upgrade and downgrade paths for `citus_update_table_statistics` + +* Fixes a bug that causes `SELECT` queries to use 2PC unnecessarily + +* Fixes a bug that might cause self-deadlocks with + `CREATE INDEX` / `REINDEX CONCURRENTLY` commands + +* Adds `citus.max_cached_connection_lifetime` GUC to set maximum connection + lifetime + +* Adds `citus.remote_copy_flush_threshold` GUC that controls + per-shard memory usages by `COPY` + +* Adds `citus_get_active_worker_nodes` UDF to deprecate + `master_get_active_worker_nodes` + +* Skips 2PC for readonly connections in a transaction + +* Makes sure that local execution starts coordinated transaction + +* Removes open temporary file warning when cancelling a query with + an open tuple store + +* Relaxes the locks when adding an existing node + diff --git a/python/tests/files/verify/expected_debian_latest_v10.0.3.txt b/python/tests/files/verify/expected_debian_latest_v10.0.3.txt new file mode 100644 index 00000000..8930cf45 --- /dev/null +++ b/python/tests/files/verify/expected_debian_latest_v10.0.3.txt @@ -0,0 +1,34 @@ +citus (10.0.3.citus-1) stable; urgency=low + + * Prevents infinite recursion for queries that involve `UNION ALL` + below `JOIN` + + * Fixes a crash in queries with a modifying `CTE` and a `SELECT` + without `FROM` + + * Fixes upgrade and downgrade paths for `citus_update_table_statistics` + + * Fixes a bug that causes `SELECT` queries to use 2PC unnecessarily + + * Fixes a bug that might cause self-deadlocks with + `CREATE INDEX` / `REINDEX CONCURRENTLY` commands + + * Adds `citus.max_cached_connection_lifetime` GUC to set maximum connection + lifetime + + * Adds `citus.remote_copy_flush_threshold` GUC that controls + per-shard memory usages by `COPY` + + * Adds `citus_get_active_worker_nodes` UDF to deprecate + `master_get_active_worker_nodes` + + * Skips 2PC for readonly connections in a transaction + + * Makes sure that local execution starts coordinated transaction + + * Removes open temporary file warning when cancelling a query with + an open tuple store + + * Relaxes the locks when adding an existing node + + -- Gurkan Indibay Thu, 18 Mar 2021 01:40:08 +0000 diff --git a/python/tests/files/verify/rpm_latest_changelog_reference.txt b/python/tests/files/verify/rpm_latest_changelog_reference.txt new file mode 100644 index 00000000..3a2f0911 --- /dev/null +++ b/python/tests/files/verify/rpm_latest_changelog_reference.txt @@ -0,0 +1,2 @@ +* Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 +- Update to Citus 10.0.3 \ No newline at end of file diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py new file mode 100644 index 00000000..cfd0a80f --- /dev/null +++ b/python/tests/test_update_package_properties.py @@ -0,0 +1,154 @@ +import os +import unittest +from datetime import datetime +from shutil import copyfile + +from .test_utils import * +from ..update_package_properties import * + +TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() +BASE_PATH = pathlib2.Path(__file__).parents[1] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") +GITHUB_TOKEN = os.getenv("GH_TOKEN") +PROJECT_VERSION = "10.0.3" if os.getenv("PROJECT_VERSION") is None else os.getenv("PROJECT_VERSION") +TAG_NAME = "v10.0.3" if os.getenv("TAG_NAME") is None else os.getenv("TAG_NAME") +PROJECT_NAME = "citus" if os.getenv("PROJECT_NAME") is None else os.getenv("PROJECT_NAME") +MICROSOFT_EMAIL = "gindibay@microsoft.com" if os.getenv("MICROSOFT_EMAIL") is None else os.getenv("MICROSOFT_EMAIL") +NAME_SURNAME = "Gurkan Indibay" if os.getenv("NAME_SURNAME") is None else os.getenv("NAME_SURNAME") +CHANGELOG_DATE = datetime.strptime('Thu, 18 Mar 2021 01:40:08 +0000', '%a, %d %b %Y %H:%M:%S %z') if os.getenv( + "CHANGELOG_DATE") is None else datetime.strptime(os.getenv("CHANGELOG_DATE"), '%a, %d %b %Y %H:%M:%S %z') + + +class PackagePropertiesTestCases(unittest.TestCase): + + def test_get_version_number(self): + self.assertEqual(get_version_number("10.0.3", True, 1), "10.0.3-1") + print(os.getenv("test")) + + def test_get_version_number_with_project_name(self): + self.assertEqual(get_version_number_with_project_name("citus", "10.0.3", True, 1), "10.0.3.citus-1") + + def test_find_nth_overlapping(self): + self.assertEqual(find_nth_overlapping("foofoo foofoo", "foofoo", 2), 7) + + def test_get_changelog_for_tag(self): + changelog = get_changelog_for_tag(GITHUB_TOKEN, "citus", "v10.0.3") + with open(f"{TEST_BASE_PATH}/files/verify/expected_changelog_10.0.3.txt", "r") as reader: + expected_changelog = reader.read() + self.assertEqual(expected_changelog, changelog) + + def test_get_debian_changelog_header(self): + header = get_debian_changelog_header("### citus v10.0.3 (March 16, 2021) ###", True, 2) + self.assertEqual(header, "citus (10.0.3.citus-2) stable; urgency=low") + + def test_get_last_changelog_from_debian(self): + refer_file_path = f"{TEST_BASE_PATH}/files/verify/debian_changelog_with_10.0.3.txt" + expected_file_path = f"{TEST_BASE_PATH}/files/verify/expected_debian_latest_v10.0.3.txt" + with open(refer_file_path, "r") as reader: + changelog = reader.read() + + latest_changelog = get_last_changelog_content_from_debian(changelog) + + with open(expected_file_path, "r") as reader: + expected_output = reader.read() + + are_strings_equal(expected_output, latest_changelog) + + def test_prepend_latest_changelog_into_debian_changelog(self): + refer_file_path = f"{TEST_BASE_PATH}/files/debian.changelog.refer" + changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" + copyfile(refer_file_path, changelog_file_path) + changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + try: + prepend_latest_changelog_into_debian_changelog(changelog, PROJECT_VERSION, True, 1, changelog_file_path, + MICROSOFT_EMAIL, NAME_SURNAME, CHANGELOG_DATE) + self.verify_prepend_debian_changelog(changelog_file_path) + finally: + os.remove(changelog_file_path) + + def verify_prepend_debian_changelog(self, changelog_file_path): + with open(changelog_file_path, "r") as reader: + content = reader.read() + latest_changelog = get_last_changelog_content_from_debian(content) + with open(f"{TEST_BASE_PATH}/files/verify/expected_debian_latest_v10.0.3.txt", "r") as reader: + expected_content = reader.read() + are_strings_equal(expected_content, latest_changelog) + + def test_convert_citus_changelog_into_rpm_changelog(self): + changelog = convert_citus_changelog_into_rpm_changelog(PROJECT_NAME, PROJECT_VERSION, MICROSOFT_EMAIL, + NAME_SURNAME, True, 1, + CHANGELOG_DATE) + with open(f"{TEST_BASE_PATH}/files/verify/rpm_latest_changelog_reference.txt", "r") as reader: + content = reader.read() + self.assertEqual(content, changelog) + + def test_update_rpm_spec(self): + project_name = "citus" + spec_file = f"{BASE_PATH}/{get_spec_file_name(project_name)}" + spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" + spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" + templates_path = f"{BASE_PATH}/templates" + copyfile(spec_file, spec_file_copy) + try: + update_rpm_spec(project_name, PROJECT_VERSION, MICROSOFT_EMAIL, NAME_SURNAME, True, 1, + spec_file, CHANGELOG_DATE, templates_path) + self.verify_rpm_spec(spec_file_reference, spec_file) + finally: + copyfile(spec_file_copy, spec_file) + os.remove(spec_file_copy) + + def verify_rpm_spec(self, spec_file_reference, spec_file_for_test): + with open(spec_file_for_test, "r") as reader_test: + with open(spec_file_reference, "r") as reader_reference: + test_str = reader_test.read() + reference_str = reader_reference.read() + are_strings_equal(reference_str, test_str) + + def test_update_pkg_vars(self): + templates_path = f"{BASE_PATH}/templates" + pkgvars_path = f"{TEST_BASE_PATH}/files/pkgvars" + pkgvars_copy_path = f"{pkgvars_path}_copy" + copyfile(pkgvars_path, pkgvars_copy_path) + try: + update_pkgvars("10.0.3", True, 1, templates_path, f"{TEST_BASE_PATH}/files/") + self.verify_pkgvars(pkgvars_path) + finally: + copyfile(pkgvars_copy_path, pkgvars_path) + os.remove(pkgvars_copy_path) + + def verify_pkgvars(self, pkgvars_path): + with open(pkgvars_path, "r") as reader: + content = reader.read() + index = content.find("pkglatest=10.0.3-1") + self.assertGreater(index, -1) + + def test_update_all_changes(self): + pkgvars_path = f"{BASE_PATH}/pkgvars" + pkgvars_copy_path = f"{pkgvars_path}_copy" + spec_file = f"{BASE_PATH}/{get_spec_file_name(PROJECT_NAME)}" + spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(PROJECT_NAME)}_copy" + spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(PROJECT_NAME)}" + + changelog_file_path = f"{BASE_PATH}/debian/changelog" + changelog_file_copy_path = f"{BASE_PATH}/debian/changelog_copy" + copyfile(changelog_file_path, changelog_file_copy_path) + copyfile(pkgvars_path, pkgvars_copy_path) + copyfile(spec_file, spec_file_copy) + + try: + update_all_changes(GITHUB_TOKEN, PROJECT_NAME, PROJECT_VERSION, TAG_NAME, True, 1, MICROSOFT_EMAIL, + NAME_SURNAME, + CHANGELOG_DATE, BASE_PATH) + self.verify_prepend_debian_changelog(changelog_file_path) + self.verify_pkgvars(pkgvars_path) + self.verify_rpm_spec(spec_file_reference, spec_file) + finally: + copyfile(changelog_file_copy_path, changelog_file_path) + copyfile(pkgvars_copy_path, pkgvars_path) + copyfile(spec_file_copy, spec_file) + + os.remove(changelog_file_copy_path) + os.remove(pkgvars_copy_path) + os.remove(spec_file_copy) + + def test_regex(self): + print(re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", "### citus v10.0.3 (March 16, 2021) ###")) diff --git a/python/tests/test_utils.py b/python/tests/test_utils.py new file mode 100644 index 00000000..e885fd82 --- /dev/null +++ b/python/tests/test_utils.py @@ -0,0 +1,10 @@ +import difflib + + +def are_strings_equal(expected_string: str, actual_str: str) -> bool: + output_list = [li for li in difflib.ndiff(expected_string, actual_str) if li[0] != ' '] + + for output in output_list: + if not (output.strip() == '+' or output.strip() == '-'): + raise Exception(f"Actual and expected string are not same Diff:{''.join(output_list)} ") + return True; diff --git a/python/tests/to_be_deleted/actual.txt b/python/tests/to_be_deleted/actual.txt new file mode 100644 index 00000000..a783f596 --- /dev/null +++ b/python/tests/to_be_deleted/actual.txt @@ -0,0 +1,315 @@ +%global pgmajorversion 11 +%global pgpackageversion 11 +%global pginstdir /usr/pgsql-%{pgpackageversion} +%global sname citus + +Summary: PostgreSQL-based distributed RDBMS +Name: %{sname}%{?pkginfix}_%{pgmajorversion} +Provides: %{sname}_%{pgmajorversion} +Conflicts: %{sname}_%{pgmajorversion} +Version: 10.0.3-1 +Release: 1%{dist} +License: AGPLv3 +Group: Applications/Databases +Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz +URL: https://github.com/citusdata/ +BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel +Requires: postgresql%{pgmajorversion}-server +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +%description +Citus horizontally scales PostgreSQL across commodity servers +using sharding and replication. Its query engine parallelizes +incoming SQL queries across these servers to enable real-time +responses on large datasets. + +Citus extends the underlying database rather than forking it, +which gives developers and enterprises the power and familiarity +of a traditional relational database. As an extension, Citus +supports new PostgreSQL releases, allowing users to benefit from +new features while maintaining compatibility with existing +PostgreSQL tools. Note that Citus supports many (but not all) SQL +commands. + +%prep +%setup -q -n %{sname}-%{version} + +%build +%configure PG_CONFIG=%{pginstdir}/bin/pg_config --with-extra-version="%{?conf_extra_version}" --with-security-flags +make %{?_smp_mflags} + +%install +%make_install +# Install documentation with a better name: +%{__mkdir} -p %{buildroot}%{pginstdir}/doc/extension +%{__cp} README.md %{buildroot}%{pginstdir}/doc/extension/README-%{sname}.md +# Set paths to be packaged other than LICENSE, README & CHANGELOG.md +echo %{pginstdir}/include/server/citus_*.h >> installation_files.list +echo %{pginstdir}/include/server/distributed/*.h >> installation_files.list +echo %{pginstdir}/lib/%{sname}.so >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}-*.sql >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list +%ifarch ppc64 ppc64le + %else + %if 0%{?rhel} && 0%{?rhel} <= 6 + %else + echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list + + # Columnar does not exist in Citus versions < 10.0 + # At this point, we don't have %{pginstdir}, + # so first check build directory for columnar. + [[ -d %{buildroot}%{pginstdir}/lib/bitcode/columnar/ ]] && echo %{pginstdir}/lib/bitcode/columnar/*.bc >> installation_files.list + %endif +%endif + +%clean +%{__rm} -rf %{buildroot} + +%files -f installation_files.list +%files +%defattr(-,root,root,-) +%doc CHANGELOG.md +%if 0%{?rhel} && 0%{?rhel} <= 6 +%doc LICENSE +%else +%license LICENSE +%endif +%doc %{pginstdir}/doc/extension/README-%{sname}.md + +%changelog +* Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 +- Official 10.0.2 release of Citus + +* Wed Jan 27 2021 - gurkanindibay 9.5.2.citus-1 +- Official 9.5.2 release of Citus + +* Tue Jan 5 2021 - gurkanindibay 9.4.4.citus-1 +- Official 9.4.4 release of Citus + +* Wed Dec 2 2020 - Onur Tirtir 9.5.1.citus-1 +- Official 9.5.1 release of Citus + +* Tue Nov 24 2020 - Onur Tirtir 9.4.3.citus-1 +- Official 9.4.3 release of Citus + +* Wed Nov 11 2020 - Onur Tirtir 9.5.0.citus-1 +- Official 9.5.0 release of Citus + +* Thu Oct 22 2020 - Onur Tirtir 9.4.2.citus-1 +- Official 9.4.2 release of Citus + +* Wed Sep 30 2020 - Onur Tirtir 9.4.1.citus-1 +- Official 9.4.1 release of Citus + +* Tue Jul 28 2020 - Onur Tirtir 9.4.0.citus-1 +- Official 9.4.0 release of Citus + +* Mon Jul 27 2020 - Onur Tirtir 9.3.5.citus-1 +- Official 9.3.5 release of Citus + +* Wed Jul 22 2020 - Onur Tirtir 9.3.4.citus-1 +- Official 9.3.4 release of Citus + +* Mon Jul 13 2020 - Onur Tirtir 9.3.3.citus-1 +- Official 9.3.3 release of Citus + +* Thu May 7 2020 - Onur Tirtir 9.3.0.citus-1 +- Update to Citus 9.3.0 + +* Tue Mar 31 2020 - Onur Tirtir 9.2.4.citus-1 +- Update to Citus 9.2.4 + +* Thu Mar 26 2020 - Onur Tirtir 9.2.3.citus-1 +- Update to Citus 9.2.3 + +* Fri Mar 6 2020 - Onur Tirtir 9.0.2.citus-1 +- Update to Citus 9.0.2 + +* Fri Mar 6 2020 - Onur Tirtir 9.2.2.citus-1 +- Update to Citus 9.2.2 + +* Fri Feb 14 2020 - Onur Tirtir 9.2.1.citus-1 +- Update to Citus 9.2.1 + +* Mon Feb 10 2020 - Onur Tirtir 9.2.0.citus-1 +- Update to Citus 9.2.0 + +* Wed Dec 18 2019 - Onur Tirtir 9.1.1.citus-1 +- Update to Citus 9.1.1 + +* Thu Nov 28 2019 - Onur Tirtir 9.1.0.citus-1 +- Update to Citus 9.1.0 + +* Wed Oct 30 2019 - Hanefi Onaldi 9.0.1.citus-1 +- Update to Citus 9.0.1 + +* Thu Oct 10 2019 - Hanefi Onaldi 9.0.0.citus-1 +- Update to Citus 9.0.0 + +* Fri Aug 9 2019 - Hanefi Onaldi 8.3.2.citus-1 +- Update to Citus 8.3.2 + +* Mon Jul 29 2019 - Hanefi Onaldi 8.3.1.citus-1 +- Update to Citus 8.3.1 + +* Wed Jul 10 2019 - Burak Velioglu 8.3.0.citus-1 +- Update to Citus 8.3.0 + +* Wed Jun 12 2019 - Burak Velioglu 8.2.2.citus-1 +- Update to Citus 8.2.2 + +* Wed Apr 3 2019 - Burak Velioglu 8.2.1.citus-1 +- Update to Citus 8.2.1 + +* Wed Apr 3 2019 - Burak Velioglu 8.1.2.citus-1 +- Update to Citus 8.1.2 + +* Thu Mar 28 2019 - Burak Velioglu 8.2.0.citus-1 +- Update to Citus 8.2.0 + +* Wed Jan 9 2019 - Burak Velioglu 8.0.3.citus-1 +- Update to Citus 8.0.3 + +* Mon Jan 7 2019 - Burak Velioglu 8.1.1.citus-1 +- Update to Citus 8.1.1 + +* Tue Dec 18 2018 - Burak Velioglu 8.1.0.citus-1 +- Update to Citus 8.1.0 + +* Thu Dec 13 2018 - Burak Velioglu 8.0.2.citus-1 +- Update to Citus 8.0.2 + +* Wed Dec 12 2018 - Burak Velioglu 7.5.4.citus-1 +- Update to Citus 7.5.4 + +* Wed Nov 28 2018 - Burak Velioglu 8.0.1.citus-1 +- Update to Citus 8.0.1 + +* Wed Nov 28 2018 - Burak Velioglu 7.5.3.citus-1 +- Update to Citus 7.5.3 + +* Wed Nov 14 2018 - Burak Velioglu 7.5.2.citus-1 +- Update to Citus 7.5.2 + +* Fri Nov 02 2018 - Burak Velioglu 8.0.0.citus-1 +- Update to Citus 8.0.0 + +* Wed Aug 29 2018 - Burak Velioglu 7.5.1.citus-1 +- Update to Citus 7.5.1 + +* Fri Jul 27 2018 - Mehmet Furkan Sahin 7.4.2.citus-1 +- Update to Citus 7.4.2 + +* Wed Jul 25 2018 - Mehmet Furkan Sahin 7.5.0.citus-1 +- Update to Citus 7.5.0 + +* Wed Jun 20 2018 - Burak Velioglu 7.4.1.citus-1 +- Update to Citus 7.4.1 + +* Thu May 17 2018 - Burak Velioglu 7.2.2.citus-1 +- Update to Citus 7.2.2 + +* Tue May 15 2018 - Burak Velioglu 7.4.0.citus-1 +- Update to Citus 7.4.0 + +* Thu Mar 15 2018 - Burak Velioglu 7.3.0.citus-1 +- Update to Citus 7.3.0 + +* Tue Feb 6 2018 - Burak Velioglu 7.2.1.citus-1 +- Update to Citus 7.2.1 + +* Tue Jan 16 2018 - Burak Velioglu 7.2.0.citus-1 +- Update to Citus 7.2.0 + +* Thu Jan 11 2018 - Burak Velioglu 6.2.5.citus-1 +- Update to Citus 6.2.5 + +* Fri Jan 05 2018 - Burak Velioglu 7.1.2.citus-1 +- Update to Citus 7.1.2 + +* Tue Dec 05 2017 - Burak Velioglu 7.1.1.citus-1 +- Update to Citus 7.1.1 + +* Wed Nov 15 2017 - Burak Velioglu 7.1.0.citus-1 +- Update to Citus 7.1.0 + +* Mon Oct 16 2017 - Burak Yucesoy 7.0.3.citus-1 +- Update to Citus 7.0.3 + +* Thu Sep 28 2017 - Burak Yucesoy 7.0.2.citus-1 +- Update to Citus 7.0.2 + +* Thu Sep 28 2017 - Burak Yucesoy 6.2.4.citus-1 +- Update to Citus 6.2.4 + +* Thu Sep 28 2017 - Burak Yucesoy 6.1.3.citus-1 +- Update to Citus 6.1.3 + +* Tue Sep 12 2017 - Burak Yucesoy 7.0.1.citus-1 +- Update to Citus 7.0.1 + +* Tue Aug 29 2017 - Burak Yucesoy 7.0.0.citus-1 +- Update to Citus 7.0.0 + +* Thu Jul 13 2017 - Burak Yucesoy 6.2.3.citus-1 +- Update to Citus 6.2.3 + +* Wed Jun 7 2017 - Burak Velioglu 6.2.2.citus-1 +- Update to Citus 6.2.2 + +* Wed Jun 7 2017 - Jason Petersen 6.1.2.citus-1 +- Update to Citus 6.1.2 + +* Wed May 24 2017 - Jason Petersen 6.2.1.citus-1 +- Update to Citus 6.2.1 + +* Tue May 16 2017 - Burak Yucesoy 6.2.0.citus-1 +- Update to Citus 6.2.0 + +* Fri May 5 2017 - Metin Doslu 6.1.1.citus-1 +- Update to Citus 6.1.1 + +* Thu Feb 9 2017 - Burak Yucesoy 6.1.0.citus-1 +- Update to Citus 6.1.0 + +* Wed Feb 8 2017 - Jason Petersen 6.0.1.citus-2 +- Transitional package to guide users to new package name + +* Wed Nov 30 2016 - Burak Yucesoy 6.0.1.citus-1 +- Update to Citus 6.0.1 + +* Tue Nov 8 2016 - Jason Petersen 6.0.0.citus-1 +- Update to Citus 6.0.0 + +* Tue Nov 8 2016 - Jason Petersen 5.2.2.citus-1 +- Update to Citus 5.2.2 + +* Tue Sep 6 2016 - Jason Petersen 5.2.1.citus-1 +- Update to Citus 5.2.1 + +* Wed Aug 17 2016 - Jason Petersen 5.2.0.citus-1 +- Update to Citus 5.2.0 + +* Mon Aug 1 2016 - Jason Petersen 5.2.0-0.1.rc.1 +- Release candidate for 5.2 + +* Fri Jun 17 2016 - Jason Petersen 5.1.1-1 +- Update to Citus 5.1.1 + +* Tue May 17 2016 - Jason Petersen 5.1.0-1 +- Update to Citus 5.1.0 + +* Mon May 16 2016 - Jason Petersen 5.1.0-0.2.rc.2 +- Fix EXPLAIN output when FORMAT JSON in use + +* Wed May 4 2016 - Jason Petersen 5.1.0-0.1.rc.1 +- Release candidate for 5.1 + +* Fri Apr 15 2016 - Jason Petersen 5.0.1-1 +- Fixes issues on 32-bit systems + +* Fri Mar 25 2016 - Devrim Gündüz 5.0.0-1 +- Initial RPM packaging for PostgreSQL RPM Repository, + based on the spec file of Jason Petersen @ Citus. diff --git a/python/tests/to_be_deleted/expected b/python/tests/to_be_deleted/expected new file mode 100644 index 00000000..1d98bfa5 --- /dev/null +++ b/python/tests/to_be_deleted/expected @@ -0,0 +1,318 @@ +%global pgmajorversion 11 +%global pgpackageversion 11 +%global pginstdir /usr/pgsql-%{pgpackageversion} +%global sname citus + +Summary: PostgreSQL-based distributed RDBMS +Name: %{sname}%{?pkginfix}_%{pgmajorversion} +Provides: %{sname}_%{pgmajorversion} +Conflicts: %{sname}_%{pgmajorversion} +Version: 10.0.3-1 +Release: 1%{dist} +License: AGPLv3 +Group: Applications/Databases +Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz +URL: https://github.com/citusdata/ +BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel +Requires: postgresql%{pgmajorversion}-server +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +%description +Citus horizontally scales PostgreSQL across commodity servers +using sharding and replication. Its query engine parallelizes +incoming SQL queries across these servers to enable real-time +responses on large datasets. + +Citus extends the underlying database rather than forking it, +which gives developers and enterprises the power and familiarity +of a traditional relational database. As an extension, Citus +supports new PostgreSQL releases, allowing users to benefit from +new features while maintaining compatibility with existing +PostgreSQL tools. Note that Citus supports many (but not all) SQL +commands. + +%prep +%setup -q -n %{sname}-%{version} + +%build +%configure PG_CONFIG=%{pginstdir}/bin/pg_config --with-extra-version="%{?conf_extra_version}" --with-security-flags +make %{?_smp_mflags} + +%install +%make_install +# Install documentation with a better name: +%{__mkdir} -p %{buildroot}%{pginstdir}/doc/extension +%{__cp} README.md %{buildroot}%{pginstdir}/doc/extension/README-%{sname}.md +# Set paths to be packaged other than LICENSE, README & CHANGELOG.md +echo %{pginstdir}/include/server/citus_*.h >> installation_files.list +echo %{pginstdir}/include/server/distributed/*.h >> installation_files.list +echo %{pginstdir}/lib/%{sname}.so >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}-*.sql >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list +%ifarch ppc64 ppc64le + %else + %if 0%{?rhel} && 0%{?rhel} <= 6 + %else + echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list + + # Columnar does not exist in Citus versions < 10.0 + # At this point, we don't have %{pginstdir}, + # so first check build directory for columnar. + [[ -d %{buildroot}%{pginstdir}/lib/bitcode/columnar/ ]] && echo %{pginstdir}/lib/bitcode/columnar/*.bc >> installation_files.list + %endif +%endif + +%clean +%{__rm} -rf %{buildroot} + +%files -f installation_files.list +%files +%defattr(-,root,root,-) +%doc CHANGELOG.md +%if 0%{?rhel} && 0%{?rhel} <= 6 +%doc LICENSE +%else +%license LICENSE +%endif +%doc %{pginstdir}/doc/extension/README-%{sname}.md + +%changelog +* Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 +- Update to Citus 10.0.3 + +* Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 +- Official 10.0.2 release of Citus + +* Wed Jan 27 2021 - gurkanindibay 9.5.2.citus-1 +- Official 9.5.2 release of Citus + +* Tue Jan 5 2021 - gurkanindibay 9.4.4.citus-1 +- Official 9.4.4 release of Citus + +* Wed Dec 2 2020 - Onur Tirtir 9.5.1.citus-1 +- Official 9.5.1 release of Citus + +* Tue Nov 24 2020 - Onur Tirtir 9.4.3.citus-1 +- Official 9.4.3 release of Citus + +* Wed Nov 11 2020 - Onur Tirtir 9.5.0.citus-1 +- Official 9.5.0 release of Citus + +* Thu Oct 22 2020 - Onur Tirtir 9.4.2.citus-1 +- Official 9.4.2 release of Citus + +* Wed Sep 30 2020 - Onur Tirtir 9.4.1.citus-1 +- Official 9.4.1 release of Citus + +* Tue Jul 28 2020 - Onur Tirtir 9.4.0.citus-1 +- Official 9.4.0 release of Citus + +* Mon Jul 27 2020 - Onur Tirtir 9.3.5.citus-1 +- Official 9.3.5 release of Citus + +* Wed Jul 22 2020 - Onur Tirtir 9.3.4.citus-1 +- Official 9.3.4 release of Citus + +* Mon Jul 13 2020 - Onur Tirtir 9.3.3.citus-1 +- Official 9.3.3 release of Citus + +* Thu May 7 2020 - Onur Tirtir 9.3.0.citus-1 +- Update to Citus 9.3.0 + +* Tue Mar 31 2020 - Onur Tirtir 9.2.4.citus-1 +- Update to Citus 9.2.4 + +* Thu Mar 26 2020 - Onur Tirtir 9.2.3.citus-1 +- Update to Citus 9.2.3 + +* Fri Mar 6 2020 - Onur Tirtir 9.0.2.citus-1 +- Update to Citus 9.0.2 + +* Fri Mar 6 2020 - Onur Tirtir 9.2.2.citus-1 +- Update to Citus 9.2.2 + +* Fri Feb 14 2020 - Onur Tirtir 9.2.1.citus-1 +- Update to Citus 9.2.1 + +* Mon Feb 10 2020 - Onur Tirtir 9.2.0.citus-1 +- Update to Citus 9.2.0 + +* Wed Dec 18 2019 - Onur Tirtir 9.1.1.citus-1 +- Update to Citus 9.1.1 + +* Thu Nov 28 2019 - Onur Tirtir 9.1.0.citus-1 +- Update to Citus 9.1.0 + +* Wed Oct 30 2019 - Hanefi Onaldi 9.0.1.citus-1 +- Update to Citus 9.0.1 + +* Thu Oct 10 2019 - Hanefi Onaldi 9.0.0.citus-1 +- Update to Citus 9.0.0 + +* Fri Aug 9 2019 - Hanefi Onaldi 8.3.2.citus-1 +- Update to Citus 8.3.2 + +* Mon Jul 29 2019 - Hanefi Onaldi 8.3.1.citus-1 +- Update to Citus 8.3.1 + +* Wed Jul 10 2019 - Burak Velioglu 8.3.0.citus-1 +- Update to Citus 8.3.0 + +* Wed Jun 12 2019 - Burak Velioglu 8.2.2.citus-1 +- Update to Citus 8.2.2 + +* Wed Apr 3 2019 - Burak Velioglu 8.2.1.citus-1 +- Update to Citus 8.2.1 + +* Wed Apr 3 2019 - Burak Velioglu 8.1.2.citus-1 +- Update to Citus 8.1.2 + +* Thu Mar 28 2019 - Burak Velioglu 8.2.0.citus-1 +- Update to Citus 8.2.0 + +* Wed Jan 9 2019 - Burak Velioglu 8.0.3.citus-1 +- Update to Citus 8.0.3 + +* Mon Jan 7 2019 - Burak Velioglu 8.1.1.citus-1 +- Update to Citus 8.1.1 + +* Tue Dec 18 2018 - Burak Velioglu 8.1.0.citus-1 +- Update to Citus 8.1.0 + +* Thu Dec 13 2018 - Burak Velioglu 8.0.2.citus-1 +- Update to Citus 8.0.2 + +* Wed Dec 12 2018 - Burak Velioglu 7.5.4.citus-1 +- Update to Citus 7.5.4 + +* Wed Nov 28 2018 - Burak Velioglu 8.0.1.citus-1 +- Update to Citus 8.0.1 + +* Wed Nov 28 2018 - Burak Velioglu 7.5.3.citus-1 +- Update to Citus 7.5.3 + +* Wed Nov 14 2018 - Burak Velioglu 7.5.2.citus-1 +- Update to Citus 7.5.2 + +* Fri Nov 02 2018 - Burak Velioglu 8.0.0.citus-1 +- Update to Citus 8.0.0 + +* Wed Aug 29 2018 - Burak Velioglu 7.5.1.citus-1 +- Update to Citus 7.5.1 + +* Fri Jul 27 2018 - Mehmet Furkan Sahin 7.4.2.citus-1 +- Update to Citus 7.4.2 + +* Wed Jul 25 2018 - Mehmet Furkan Sahin 7.5.0.citus-1 +- Update to Citus 7.5.0 + +* Wed Jun 20 2018 - Burak Velioglu 7.4.1.citus-1 +- Update to Citus 7.4.1 + +* Thu May 17 2018 - Burak Velioglu 7.2.2.citus-1 +- Update to Citus 7.2.2 + +* Tue May 15 2018 - Burak Velioglu 7.4.0.citus-1 +- Update to Citus 7.4.0 + +* Thu Mar 15 2018 - Burak Velioglu 7.3.0.citus-1 +- Update to Citus 7.3.0 + +* Tue Feb 6 2018 - Burak Velioglu 7.2.1.citus-1 +- Update to Citus 7.2.1 + +* Tue Jan 16 2018 - Burak Velioglu 7.2.0.citus-1 +- Update to Citus 7.2.0 + +* Thu Jan 11 2018 - Burak Velioglu 6.2.5.citus-1 +- Update to Citus 6.2.5 + +* Fri Jan 05 2018 - Burak Velioglu 7.1.2.citus-1 +- Update to Citus 7.1.2 + +* Tue Dec 05 2017 - Burak Velioglu 7.1.1.citus-1 +- Update to Citus 7.1.1 + +* Wed Nov 15 2017 - Burak Velioglu 7.1.0.citus-1 +- Update to Citus 7.1.0 + +* Mon Oct 16 2017 - Burak Yucesoy 7.0.3.citus-1 +- Update to Citus 7.0.3 + +* Thu Sep 28 2017 - Burak Yucesoy 7.0.2.citus-1 +- Update to Citus 7.0.2 + +* Thu Sep 28 2017 - Burak Yucesoy 6.2.4.citus-1 +- Update to Citus 6.2.4 + +* Thu Sep 28 2017 - Burak Yucesoy 6.1.3.citus-1 +- Update to Citus 6.1.3 + +* Tue Sep 12 2017 - Burak Yucesoy 7.0.1.citus-1 +- Update to Citus 7.0.1 + +* Tue Aug 29 2017 - Burak Yucesoy 7.0.0.citus-1 +- Update to Citus 7.0.0 + +* Thu Jul 13 2017 - Burak Yucesoy 6.2.3.citus-1 +- Update to Citus 6.2.3 + +* Wed Jun 7 2017 - Burak Velioglu 6.2.2.citus-1 +- Update to Citus 6.2.2 + +* Wed Jun 7 2017 - Jason Petersen 6.1.2.citus-1 +- Update to Citus 6.1.2 + +* Wed May 24 2017 - Jason Petersen 6.2.1.citus-1 +- Update to Citus 6.2.1 + +* Tue May 16 2017 - Burak Yucesoy 6.2.0.citus-1 +- Update to Citus 6.2.0 + +* Fri May 5 2017 - Metin Doslu 6.1.1.citus-1 +- Update to Citus 6.1.1 + +* Thu Feb 9 2017 - Burak Yucesoy 6.1.0.citus-1 +- Update to Citus 6.1.0 + +* Wed Feb 8 2017 - Jason Petersen 6.0.1.citus-2 +- Transitional package to guide users to new package name + +* Wed Nov 30 2016 - Burak Yucesoy 6.0.1.citus-1 +- Update to Citus 6.0.1 + +* Tue Nov 8 2016 - Jason Petersen 6.0.0.citus-1 +- Update to Citus 6.0.0 + +* Tue Nov 8 2016 - Jason Petersen 5.2.2.citus-1 +- Update to Citus 5.2.2 + +* Tue Sep 6 2016 - Jason Petersen 5.2.1.citus-1 +- Update to Citus 5.2.1 + +* Wed Aug 17 2016 - Jason Petersen 5.2.0.citus-1 +- Update to Citus 5.2.0 + +* Mon Aug 1 2016 - Jason Petersen 5.2.0-0.1.rc.1 +- Release candidate for 5.2 + +* Fri Jun 17 2016 - Jason Petersen 5.1.1-1 +- Update to Citus 5.1.1 + +* Tue May 17 2016 - Jason Petersen 5.1.0-1 +- Update to Citus 5.1.0 + +* Mon May 16 2016 - Jason Petersen 5.1.0-0.2.rc.2 +- Fix EXPLAIN output when FORMAT JSON in use + +* Wed May 4 2016 - Jason Petersen 5.1.0-0.1.rc.1 +- Release candidate for 5.1 + +* Fri Apr 15 2016 - Jason Petersen 5.0.1-1 +- Fixes issues on 32-bit systems + +* Fri Mar 25 2016 - Devrim Gündüz 5.0.0-1 +- Initial RPM packaging for PostgreSQL RPM Repository, + based on the spec file of Jason Petersen @ Citus. diff --git a/python/update_package_properties.py b/python/update_package_properties.py new file mode 100644 index 00000000..e85131f8 --- /dev/null +++ b/python/update_package_properties.py @@ -0,0 +1,319 @@ +import argparse +import re +from datetime import date + +import pathlib2 +import string_utils +from github import Github +from jinja2 import Environment, FileSystemLoader +from parameters_validation import no_whitespaces, non_blank, non_empty, non_negative, validate_parameters, \ + parameter_validation + +BASE_PATH = pathlib2.Path(__file__).parent.absolute() + + +@parameter_validation +def is_version(version: str): + if version is None or not version: + raise ValueError("version should be non-empty and should not be None") + if not re.match(r"\d+\.\d+\.\d+$", version): + raise ValueError( + "version should include 3 levels of versions consists of numbers separated with dots. e.g: 10.0.1") + + +@parameter_validation +def is_tag(tag: str): + if tag is None or not tag: + raise ValueError("tag should be non-empty and should not be None") + if not re.match(r"v\d+\.\d+\.\d+$", tag): + raise ValueError( + "tag should start with v and should include 3 levels of versions consists of numbers " + + "separated with dots. e.g: v10.0.1") + + +@parameter_validation +def is_email(email: str): + if not string_utils.is_email(email): + raise ValueError("Parameter is not in email format") + + +@parameter_validation +def is_project_changelog_header(header: str): + if header is None or not header: + raise ValueError("header should be non-empty and should not be None") + if not re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", header): + raise ValueError( + f"changelog header is in invalid format. Actual:{header} Expected: ### citus v8.3.3 (March 23, 2021) ### ") + + +def get_spec_file_name(project_name: str) -> str: + return f"{project_name}.spec" + + +def get_version_number(version: str, fancy: bool, fancy_release_count: int) -> str: + fancy_suffix = f"-{fancy_release_count}" if fancy else "" + return f"{version}{fancy_suffix}" + + +def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, fancy_release_count: int) -> str: + fancy_suffix = f"-{fancy_release_count}" if fancy else "" + return f"{version}.{project_name}{fancy_suffix}" + + +def get_template_environment(template_dir: str) -> Environment: + file_loader = FileSystemLoader(template_dir) + env = Environment(loader=file_loader) + return env + + +def find_nth_overlapping(subject_string, search_string, n) -> int: + start = subject_string.find(search_string) + + while start >= 0 and n > 1: + start = subject_string.find(search_string, start + 1) + n -= 1 + return start + + +def find_nth_overlapping_line_by_regex(subject_string, regex_pattern, n) -> int: + lines = subject_string.splitlines() + counter = 0 + index = -1 + for i in range(len(lines)): + if re.match(regex_pattern, lines[i]): + counter = counter + 1 + if counter == n: + index = i + break + return index + + +# def find_nth_overlapping_regex(subject_string, search_string, n): +# re. +# start = subject_string.find(search_string) +# +# while start >= 0 and n > 1: +# start = subject_string.find(search_string, start + 1) +# n -= 1 +# return start + + +def get_last_changelog_content(all_changelog_content: str) -> str: + second_changelog_index = find_nth_overlapping(all_changelog_content, "###", 3) + changelogs = all_changelog_content[:second_changelog_index] + lines = changelogs.splitlines() + if len(lines) < 1: + raise Exception("At least one line should be in changelog") + changelog_header = lines[0] + if not changelog_header.startswith("###"): + raise Exception("Changelog header should start with '###'") + return changelogs + + +def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: + second_changelog_index = find_nth_overlapping_line_by_regex(all_changelog_content, "^[a-zA-Z]", 2) + lines = all_changelog_content.splitlines() + changelogs = "".join([str(elem) + "\n" for elem in lines[:second_changelog_index - 1]]) + if len(lines) < 1: + raise Exception("At least one line should be in changelog") + return changelogs + + +def remove_parentheses_from_string(param: str) -> str: + return re.sub(r"[(\[].*?[)\]]", "", param) + + +def get_changelog_for_tag(github_token: str, project_name: str, tag_name: str) -> str: + g = Github(github_token) + repo = g.get_repo(f"citusdata/{project_name}") + all_changelog_content = repo.get_contents("CHANGELOG.md", ref=tag_name) + last_changelog_content = get_last_changelog_content(all_changelog_content.decoded_content.decode()) + return last_changelog_content + + +# truncates # chars , get the version an put parentheses around version number adds 'stable; urgency=low' at the end +# changelog_header=> ### citus v8.3.3 (March 23, 2021) ### +# debian header => citus (10.0.3.citus-1) stable; urgency=low +@validate_parameters +def get_debian_changelog_header(changelog_header: is_project_changelog_header(str), fancy: bool, + fancy_version_number: int) -> str: + hash_removed_string = changelog_header.removeprefix("### ").removesuffix(" ###") + parentheses_removed_string = remove_parentheses_from_string(hash_removed_string) + words = parentheses_removed_string.strip().split(" ") + if len(words) != 2: + raise Exception("Two words should be included in striped version header") + project_name = words[0] + project_version = words[1].removeprefix("v") + version_on_changelog = get_version_number_with_project_name(project_name, project_version, fancy, + fancy_version_number) + + return f"{project_name} ({version_on_changelog}) stable; urgency=low" + + +def convert_citus_changelog_into_debian_changelog(citus_changelog: str, fancy: bool, + fancy_version_number: int, microsoft_email: str, name_surname: str, + changelog_date) -> str: + lines = citus_changelog.splitlines() + lines[0] = get_debian_changelog_header(lines[0], fancy, fancy_version_number) + lines.append(get_debian_trailer(microsoft_email, name_surname, changelog_date)) + debian_latest_changelog = "" + for i in range(len(lines)): + append_line = lines[i] if i == 0 or i == len(lines) - 1 else ' ' + lines[i] + debian_latest_changelog = debian_latest_changelog + append_line + "\n" + return debian_latest_changelog + + +def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, tag_name: str, fancy: bool, + fancy_version_number: int, changelog_file_path: str, + microsoft_email: str, name_surname: str, + changelog_date: date) -> None: + debian_latest_changelog = convert_citus_changelog_into_debian_changelog(latest_changelog, fancy, + fancy_version_number, microsoft_email, + name_surname, changelog_date) + with open(changelog_file_path, "r+") as reader: + if not (f"({tag_name}" in reader.readline()): + reader.seek(0, 0) + old_changelog = reader.read() + changelog = f"{debian_latest_changelog}{old_changelog}" + reader.seek(0, 0) + reader.write(changelog) + else: + print("Already version in the debian changelog") + + +# print(get_last_changelog_content( +# "### citus v8.3.3 (March 23, 2021) ###\n\n * Fix for something \n\n### citus v8.3.2 (March 23, 2020) ###\n\n* Fix for another thing\n\n")) + + +# print(change_header_into_debian_header("### citus v8.3.3 (March 23, 2021) ###")) +# print(remove_parentheses_from_string("### citus v8.3.3 (March 23, 2021) ###")) + +# print("citus v8.3.3".split(r" ")) +# print(get_changelog_for_tag("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3")) +# print(get_debian_changelog_for_latest_entry("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3")) +# print(save_latest_changelog_into_debian_template("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3")) + + +@validate_parameters +def update_pkgvars(version: is_version(non_empty(no_whitespaces(non_blank(str)))), fancy: bool, + fancy_release_count: non_negative(int), templates_path: str, pkgvars_path: str) -> None: + env = get_template_environment(templates_path) + + version_str = get_version_number(version, fancy, fancy_release_count) + + template = env.get_template('pkgvars.tmpl') + + pkgvars_content = template.render(version=version_str) + with open(f'{pkgvars_path}/pkgvars', "w") as writer: + writer.write(pkgvars_content) + + +def get_rpm_changelog_history(project_name: str, spec_file_path: str) -> str: + with open(spec_file_path, "r") as reader: + spec_content = reader.read() + changelog_index = spec_content.find("%changelog") + changelog_content = spec_content[changelog_index + len("%changelog") + 1:] + + return changelog_content + + +def get_rpm_header(project_name: str, microsoft_email: str, name_surname: str, project_version: str, + is_fancy_versioned: bool, + fancy_version_number: int, changelog_date: date): + formatted_date = changelog_date.strftime("%a %b %d %Y") + return f"* {formatted_date} - {name_surname} <{microsoft_email}> {get_version_number_with_project_name(project_name, project_version, is_fancy_versioned, fancy_version_number)} " + + +def get_debian_trailer(microsoft_email: str, name_surname: str, changelog_date: date): + formatted_date = changelog_date.strftime("%a, %d %b %Y %H:%M:%S %z ") + return f" -- {name_surname} <{microsoft_email}> {formatted_date} \n " + + +def get_rpm_changelog_from_actual_changelog(project_name: str, project_version: str, latest_changelog: str, + microsoft_email: str, + name_surname: str, fancy: bool, fancy_version_number: int, + spec_file_path: str): + rpm_changelog_history = get_rpm_changelog_history(project_name, spec_file_path) + changelog_histories = rpm_changelog_history.splitlines() + rpm_changelog = "" + if len(changelog_histories) < 1 or not (project_version in changelog_histories[1]): + lines = latest_changelog.splitlines() + lines[0] = get_rpm_header(project_name, microsoft_email, name_surname, project_version, fancy, + fancy_version_number, date.today()) + rpm_changelog = rpm_changelog.join([str(elem) + "\n" for elem in lines]) + rpm_changelog = f"{rpm_changelog}{rpm_changelog_history}" + else: + rpm_changelog = rpm_changelog_history + + return rpm_changelog + + +def convert_citus_changelog_into_rpm_changelog(project_name: str, project_version: str, microsoft_email: str, + name_surname: str, fancy: bool, fancy_version_number: int, + changelog_date: date) -> str: + header = get_rpm_header(project_name, microsoft_email, name_surname, project_version, fancy, + fancy_version_number, changelog_date) + rpm_changelog = f"{header.strip()}\n- Update to {project_name.capitalize()} {project_version}" + + return rpm_changelog + + +# # print(get_rpm_changelog_history("citus")) +# changelog = get_changelog_for_tag("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3") +# print(get_rpm_changelog("citus", "10.0.3", changelog, "gindibay@microsoft.com", "Gürkan İndibay", True, 1)) +# print(get_rpm_header("citus", "gindibay@microsoft.com", "Gurkan Indibay", "10.0.3", True, 2)) + + +def update_rpm_spec(project_name: str, project_version: str, microsoft_email: str, + name_surname: str, fancy: bool, fancy_version_number: int, spec_file_name: str, + changelog_date: date, templates_path: str) -> None: + env = get_template_environment(templates_path) + + fancy_version_str = get_version_number(project_version, fancy, fancy_version_number) + template = env.get_template('project.spec.tmpl') + rpm_changelog_history = get_rpm_changelog_history(project_name, spec_file_name) + + latest_changelog = convert_citus_changelog_into_rpm_changelog(project_name, project_version, microsoft_email, + name_surname, fancy, fancy_version_number, + changelog_date) + changelog = f"{latest_changelog}\n\n{rpm_changelog_history}" + content = template.render(version=project_version, fancy_version_str=fancy_version_str, + fancy_version_no=fancy_version_number, changelog=changelog) + with open(spec_file_name, "w+") as writer: + writer.write(content) + + +@validate_parameters +def update_all_changes(github_token: non_empty(str), project_name: non_empty(str), project_version: is_version(str), + tag_name: is_tag(str), fancy: bool, fancy_version_number: non_negative(int), + microsoft_email: is_email(str), + name_surname: non_empty(non_blank(str)), release_date: date, packaging_path: str): + templates_path = f"{BASE_PATH}/templates" + update_pkgvars(project_version, fancy, fancy_version_number, templates_path, f"{packaging_path}") + latest_changelog = get_changelog_for_tag(github_token, project_name, tag_name) + prepend_latest_changelog_into_debian_changelog(latest_changelog, tag_name, fancy, fancy_version_number, + f"{packaging_path}/debian/changelog", microsoft_email, name_surname, + release_date) + spec_file_name = f"{packaging_path}/{get_spec_file_name(project_name)}" + update_rpm_spec(project_name, project_version, microsoft_email, name_surname, fancy, fancy_version_number, + spec_file_name, release_date, templates_path) + + +# update_all_changes("93647419346e194ae3094598139d68ebf2263ee0", "citus", "10.0.3", "v10.0.3", True, 1, +# "gindibay@microsoft.com", "Gurkan Indibay") +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--gh_token') + parser.add_argument('--prj_name') + parser.add_argument('--prj_ver') + parser.add_argument('--tag_name') + parser.add_argument('--fancy') + parser.add_argument('--fancy_ver_no') + parser.add_argument('--email') + parser.add_argument('--name') + parser.add_argument('--date') + parser.add_argument('--exec_path') + args = parser.parse_args() + + update_all_changes(args.gh_token, args.prj_name, args.tag_name, args.fancy, args.fancy_ver_no, args.email, + args.name, args.date, args.exec_path) diff --git a/python/upload_to_package_cloud.py b/python/upload_to_package_cloud.py new file mode 100644 index 00000000..cee4c94a --- /dev/null +++ b/python/upload_to_package_cloud.py @@ -0,0 +1,110 @@ +import os +import sys +import ntpath; +from dataclasses import dataclass +from typing import List + +import requests + +supported_distros = { + "el/7": 140, + "el/8": 205, + "ol/7": 146, + "debian/buster": 150, + "debian/stretch": 149, + "ubuntu/focal": 210, + "ubuntu/bionic": 190, + "ubuntu/xenial": 165 +} + + +@dataclass +class ReturnValue: + success_status: bool + message: str + file_name: str + distro: str + repo: str + + +@dataclass +class MultipleReturnValue: + def __init__(self, ret_vals: List[ReturnValue]): + self.return_values = ret_vals + + multiple_return_value: List[ReturnValue] + + def success_status(self) -> bool: + return len([r for r in self.return_values if not r.success_status]) == 0 + + +def upload_to_packagecloud(distro_name, package_name, packagecloud_token, repo_name) -> ReturnValue: + distro_id = supported_distros[distro_name] + files = { + 'package[distro_version_id]': (None, str(distro_id)), + 'package[package_file]': ( + package_name, open(package_name, 'rb')), + } + + package_query_url = 'https://' + packagecloud_token + ':@packagecloud.io/api/v1/repos/citus-bot/' + repo_name + '/packages.json' + print(f"Uploading package {ntpath.basename(package_name)} using path {package_query_url}") + response = requests.post(package_query_url, files=files) + print(f"Response from package cloud: {response.content}") + return ReturnValue(response.ok, response.content, package_name, distro_name, repo_name) + + +def upload_files_in_directory_to_packagecloud(directoryName: str, distro_name: str, package_cloud_token: str, + repo_name: str) -> MultipleReturnValue: + # print("Distro Name: " + distro_name) + # print("Supported Distros:") + for key, value in supported_distros.items(): + print(key + "=>" + str(value)) + ret_status: List[ReturnValue] = [] + + # TODO may be parameterized to push all files. Now only two level + print("Test version") + for firstLevelFileItem in os.listdir(directoryName): + item_name = os.path.join(directoryName, firstLevelFileItem) + if os.path.isdir(item_name): + for filename in os.listdir(item_name): + if filename.lower().endswith((".rpm", ".deb")): + ret_val = upload_to_packagecloud(distro_name, os.path.join(item_name, filename), + package_cloud_token, + repo_name) + ret_status.append(ret_val) + else: + filename = firstLevelFileItem + if filename.lower().endswith((".rpm", ".deb")): + ret_val = upload_to_packagecloud(distro_name, os.path.join(directoryName, filename), package_cloud_token, + repo_name) + ret_status.append(ret_val) + + return MultipleReturnValue(ret_status) + + +if len(sys.argv) < 3: + raise Exception("Distro Name package_cloud_api_token and repository name parameters should be provided") + +target_platform = sys.argv[1] +package_cloud_api_token = sys.argv[2] +repository_name = sys.argv[3] +multiple_return_value = upload_files_in_directory_to_packagecloud(os.path.join(os.getcwd(), "pkgs/releases"), + target_platform, + package_cloud_api_token, repository_name) +print(multiple_return_value.success_status()) +print(multiple_return_value.return_values) +for rv in multiple_return_value.return_values: + if not rv.success_status: + print( + f'Error occured while uploading file on package cloud. Error: {rv.message} Distro: {rv.distro} File Name: {ntpath.basename(rv.file_name)} Repo Name: {rv.repo}') + else: + print( + f'File successfully uploaded. Distro: {rv.distro} File Name: {ntpath.basename(rv.file_name)} Repo Name: {rv.repo}') +if multiple_return_value.success_status(): + sys.exit(0) +else: + sys.exit(1) +# for rv in multiple_return_value.return_values: +# if(rv.) +# print(r.message) +# print(r.success_status) From 27f2c8804bfd393effc8029184fa8d18768ea113 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 31 Mar 2021 22:33:30 +0300 Subject: [PATCH 004/106] Remove github token from scripts --- python/update_package_properties.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index e85131f8..c401dbb2 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -181,18 +181,6 @@ def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, tag_na print("Already version in the debian changelog") -# print(get_last_changelog_content( -# "### citus v8.3.3 (March 23, 2021) ###\n\n * Fix for something \n\n### citus v8.3.2 (March 23, 2020) ###\n\n* Fix for another thing\n\n")) - - -# print(change_header_into_debian_header("### citus v8.3.3 (March 23, 2021) ###")) -# print(remove_parentheses_from_string("### citus v8.3.3 (March 23, 2021) ###")) - -# print("citus v8.3.3".split(r" ")) -# print(get_changelog_for_tag("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3")) -# print(get_debian_changelog_for_latest_entry("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3")) -# print(save_latest_changelog_into_debian_template("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3")) - @validate_parameters def update_pkgvars(version: is_version(non_empty(no_whitespaces(non_blank(str)))), fancy: bool, From 5fcd72bb4e69fec508503baf1771f2b7455d17b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 31 Mar 2021 22:39:26 +0300 Subject: [PATCH 005/106] Fix requirements.txt path --- .github/workflows/tool-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 10894a3a..1e1f0eae 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -33,7 +33,7 @@ jobs: - name: Run a multi-line script run: | ls -la - pip install -r requirements.txt + pip install -r python/requirements.txt export GH_TOKEN=$(GH_TOKEN) export BASE_PATH=$(pwd) python -m unittest ./python/tests/test_update_package_properties.py From 2eae7db713eed50290f92faaaec6c961f89742d6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 31 Mar 2021 23:37:06 +0300 Subject: [PATCH 006/106] Remove unattended upgrades since gives error --- python/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index 46304c9e..cb33ace4 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -65,7 +65,6 @@ systemd-python==234 Twisted==18.9.0 ubuntu-advantage-tools==20.3 ufw==0.36 -unattended-upgrades==0.1 urllib3==1.25.8 virtualenv==20.0.17 wadllib==1.3.3 From ffeeaa4370517e1147888ce4fa29bc2360393703 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 31 Mar 2021 23:39:27 +0300 Subject: [PATCH 007/106] Remove unneccessary pkgvars file --- python/pkgvars | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 python/pkgvars diff --git a/python/pkgvars b/python/pkgvars deleted file mode 100644 index 2ad71e5c..00000000 --- a/python/pkgvars +++ /dev/null @@ -1,6 +0,0 @@ -pkgname=citus -pkgdesc='Citus (Open-Source)' -pkglatest=10.0.3-1 -releasepg=11,12,13 -nightlypg=12,13 -versioning=fancy \ No newline at end of file From 02f4452ba317f00cdd77da80b8b4318a68ac4ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 31 Mar 2021 23:48:22 +0300 Subject: [PATCH 008/106] Update tool-tests.yml --- .github/workflows/tool-tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 1e1f0eae..721d3f04 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -26,14 +26,14 @@ jobs: - uses: actions/checkout@v2 # Runs a single command using the runners shell + - name: ls + run: ls -la + - name: GH-TOKEN Export + run: export GH_TOKEN=$(GH_TOKEN) + - name: install requirements + run: python -m pip install -r requirements.txt - name: Run a one-line script - run: echo Hello, world! - - # Runs a set of commands using the runners shell - - name: Run a multi-line script - run: | - ls -la - pip install -r python/requirements.txt - export GH_TOKEN=$(GH_TOKEN) - export BASE_PATH=$(pwd) - python -m unittest ./python/tests/test_update_package_properties.py + run: python -m unittest ./python/tests/test_update_package_properties.py + + + From 271e0d49b285bf9787abb5a4617744fdc6cc8245 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 31 Mar 2021 23:50:07 +0300 Subject: [PATCH 009/106] Update requirements path --- .github/workflows/tool-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 721d3f04..225d2f64 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -31,7 +31,7 @@ jobs: - name: GH-TOKEN Export run: export GH_TOKEN=$(GH_TOKEN) - name: install requirements - run: python -m pip install -r requirements.txt + run: python -m pip install -r python/requirements.txt - name: Run a one-line script run: python -m unittest ./python/tests/test_update_package_properties.py From 6f2f69213130a45a6899972e12a8ca35c7d6da31 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 00:13:38 +0300 Subject: [PATCH 010/106] Update test path --- .github/workflows/tool-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 225d2f64..1ca25b18 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -33,7 +33,7 @@ jobs: - name: install requirements run: python -m pip install -r python/requirements.txt - name: Run a one-line script - run: python -m unittest ./python/tests/test_update_package_properties.py + run: python -m unittest $(pwd)/python/tests/test_update_package_properties.py From 0d5079a84bb19298e34d30c672a67b5017fbc4cb Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 00:22:14 +0300 Subject: [PATCH 011/106] Add missing requirements --- python/requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/requirements.txt b/python/requirements.txt index cb33ace4..afb1c4e6 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -13,6 +13,7 @@ configobj==5.0.6 constantly==15.1.0 cryptography==2.8 dbus-python==1.2.16 +Deprecated==1.2.12 distlib==0.3.0 distro==1.4.0 distro-info===0.23ubuntu1 @@ -36,10 +37,12 @@ MarkupSafe==1.1.0 more-itertools==4.2.0 netifaces==0.10.4 oauthlib==3.1.0 +parameters-validation==1.2.0 pathlib2==2.3.5 pexpect==4.6.0 pyasn1==0.4.2 pyasn1-modules==0.2.1 +PyGithub==1.54.1 Pygments==2.3.1 PyGObject==3.36.0 PyHamcrest==1.9.0 @@ -52,6 +55,7 @@ pyserial==3.4 python-apt==2.0.0+ubuntu0.20.4.4 python-dateutil==2.7.3 python-debian===0.1.36ubuntu1 +python-string-utils==1.0.0 PyYAML==5.3.1 requests==2.22.0 requests-unixsocket==0.2.0 @@ -65,8 +69,10 @@ systemd-python==234 Twisted==18.9.0 ubuntu-advantage-tools==20.3 ufw==0.36 +unattended-upgrades==0.1 urllib3==1.25.8 virtualenv==20.0.17 wadllib==1.3.3 +wrapt==1.12.1 zipp==1.0.0 zope.interface==4.7.1 From 09fcf54df4c4506864295707dd63e5aefd6e71d6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 00:23:42 +0300 Subject: [PATCH 012/106] Remove missing package from requirements --- python/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index afb1c4e6..0106dbd2 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -69,7 +69,6 @@ systemd-python==234 Twisted==18.9.0 ubuntu-advantage-tools==20.3 ufw==0.36 -unattended-upgrades==0.1 urllib3==1.25.8 virtualenv==20.0.17 wadllib==1.3.3 From eddb07827b2a374b6f39e076bc5ddbc527395b8a Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 00:37:48 +0300 Subject: [PATCH 013/106] Fix Paths for test Move changelog into test dir --- python/{ => tests/files}/debian/changelog | 0 python/tests/test_update_package_properties.py | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) rename python/{ => tests/files}/debian/changelog (100%) diff --git a/python/debian/changelog b/python/tests/files/debian/changelog similarity index 100% rename from python/debian/changelog rename to python/tests/files/debian/changelog diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index cfd0a80f..d8416bd7 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -83,7 +83,7 @@ def test_convert_citus_changelog_into_rpm_changelog(self): def test_update_rpm_spec(self): project_name = "citus" - spec_file = f"{BASE_PATH}/{get_spec_file_name(project_name)}" + spec_file = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" templates_path = f"{BASE_PATH}/templates" @@ -122,14 +122,14 @@ def verify_pkgvars(self, pkgvars_path): self.assertGreater(index, -1) def test_update_all_changes(self): - pkgvars_path = f"{BASE_PATH}/pkgvars" + pkgvars_path = f"{TEST_BASE_PATH}/files/pkgvars" pkgvars_copy_path = f"{pkgvars_path}_copy" - spec_file = f"{BASE_PATH}/{get_spec_file_name(PROJECT_NAME)}" - spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(PROJECT_NAME)}_copy" + spec_file = f"{TEST_BASE_PATH}/files/{get_spec_file_name(PROJECT_NAME)}" + spec_file_copy = f"{spec_file}_copy" spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(PROJECT_NAME)}" - changelog_file_path = f"{BASE_PATH}/debian/changelog" - changelog_file_copy_path = f"{BASE_PATH}/debian/changelog_copy" + changelog_file_path = f"{TEST_BASE_PATH}/files/debian/changelog" + changelog_file_copy_path = f"{changelog_file_path}_copy" copyfile(changelog_file_path, changelog_file_copy_path) copyfile(pkgvars_path, pkgvars_copy_path) copyfile(spec_file, spec_file_copy) @@ -137,7 +137,7 @@ def test_update_all_changes(self): try: update_all_changes(GITHUB_TOKEN, PROJECT_NAME, PROJECT_VERSION, TAG_NAME, True, 1, MICROSOFT_EMAIL, NAME_SURNAME, - CHANGELOG_DATE, BASE_PATH) + CHANGELOG_DATE, f"{TEST_BASE_PATH}/files") self.verify_prepend_debian_changelog(changelog_file_path) self.verify_pkgvars(pkgvars_path) self.verify_rpm_spec(spec_file_reference, spec_file) From e8318adbc97602f7f8303144dbc90657ce1a056a Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 00:41:58 +0300 Subject: [PATCH 014/106] Install python 3.9 --- .github/workflows/tool-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 1ca25b18..390155a7 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -26,6 +26,8 @@ jobs: - uses: actions/checkout@v2 # Runs a single command using the runners shell + - name: Install python dependencies + run: sudo apt install software-properties-common && sudo add-apt-repository ppa:deadsnakes/ppa && sudo apt install python3.9 && python3.9 --version && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 - name: ls run: ls -la - name: GH-TOKEN Export From a98f4480cd065e40b27ca27673922851d9a401bc Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 00:44:57 +0300 Subject: [PATCH 015/106] Install python 3.9 --- .github/workflows/tool-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 390155a7..4554c79d 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -27,7 +27,7 @@ jobs: # Runs a single command using the runners shell - name: Install python dependencies - run: sudo apt install software-properties-common && sudo add-apt-repository ppa:deadsnakes/ppa && sudo apt install python3.9 && python3.9 --version && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 + run: python --version - name: ls run: ls -la - name: GH-TOKEN Export From 28ffcd19f2421641aa539834402250fc9d510b7d Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 08:10:55 +0300 Subject: [PATCH 016/106] Change removeprefix and removesuffix since this features are python 3.9 features but GithubActions environment is 3.8.5 --- python/update_package_properties.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index c401dbb2..9f7248fa 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -137,13 +137,13 @@ def get_changelog_for_tag(github_token: str, project_name: str, tag_name: str) - @validate_parameters def get_debian_changelog_header(changelog_header: is_project_changelog_header(str), fancy: bool, fancy_version_number: int) -> str: - hash_removed_string = changelog_header.removeprefix("### ").removesuffix(" ###") + hash_removed_string = changelog_header.lstrip("### ").rstrip(" ###") parentheses_removed_string = remove_parentheses_from_string(hash_removed_string) words = parentheses_removed_string.strip().split(" ") if len(words) != 2: raise Exception("Two words should be included in striped version header") project_name = words[0] - project_version = words[1].removeprefix("v") + project_version = words[1].lstrip("v") version_on_changelog = get_version_number_with_project_name(project_name, project_version, fancy, fancy_version_number) From 9189081ce8587462d6619fc1f762b44ed610d1f3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 08:14:48 +0300 Subject: [PATCH 017/106] Add validation for GITHUB_TOKEN --- python/update_package_properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 9f7248fa..9c28299a 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -272,7 +272,7 @@ def update_rpm_spec(project_name: str, project_version: str, microsoft_email: st @validate_parameters -def update_all_changes(github_token: non_empty(str), project_name: non_empty(str), project_version: is_version(str), +def update_all_changes(github_token: non_blank(non_empty(str)), project_name: non_empty(str), project_version: is_version(str), tag_name: is_tag(str), fancy: bool, fancy_version_number: non_negative(int), microsoft_email: is_email(str), name_surname: non_empty(non_blank(str)), release_date: date, packaging_path: str): From b34d6e50a682717c3fe402d54c978f35f31cec36 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 08:16:32 +0300 Subject: [PATCH 018/106] Fix validation for GITHUB_TOKEN --- python/update_package_properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 9c28299a..06636f91 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -272,7 +272,7 @@ def update_rpm_spec(project_name: str, project_version: str, microsoft_email: st @validate_parameters -def update_all_changes(github_token: non_blank(non_empty(str)), project_name: non_empty(str), project_version: is_version(str), +def update_all_changes(github_token: non_empty(non_blank(str)), project_name: non_empty(str), project_version: is_version(str), tag_name: is_tag(str), fancy: bool, fancy_version_number: non_negative(int), microsoft_email: is_email(str), name_surname: non_empty(non_blank(str)), release_date: date, packaging_path: str): From cb8390180e3fbdcd01d019cefb336c114dee96b0 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 08:28:45 +0300 Subject: [PATCH 019/106] Add parameter GH_TOKEN into pipeline yml --- .github/workflows/tool-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 4554c79d..4987ead9 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -2,6 +2,10 @@ name: CI +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + + # Controls when the action will run. on: # Triggers the workflow on push or pull request events but only for the develop branch @@ -30,8 +34,6 @@ jobs: run: python --version - name: ls run: ls -la - - name: GH-TOKEN Export - run: export GH_TOKEN=$(GH_TOKEN) - name: install requirements run: python -m pip install -r python/requirements.txt - name: Run a one-line script From a325539b40ea48db61e0d2a57862692b320d5980 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 08:40:13 +0300 Subject: [PATCH 020/106] Remove logs and fix name for the last task --- .github/workflows/tool-tests.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 4987ead9..5086d15c 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -29,14 +29,9 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - # Runs a single command using the runners shell - - name: Install python dependencies - run: python --version - - name: ls - run: ls -la - name: install requirements run: python -m pip install -r python/requirements.txt - - name: Run a one-line script + - name: Execute Unit tests run: python -m unittest $(pwd)/python/tests/test_update_package_properties.py From 476ab31142b623af660bbe7288e26fc7a123cbd8 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 12:16:44 +0300 Subject: [PATCH 021/106] Fix missing parameter problems on update_al_changes --- bash/execute_update_properties.sh | 11 ++--------- python/update_package_properties.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/bash/execute_update_properties.sh b/bash/execute_update_properties.sh index aab50c8b..721d6447 100644 --- a/bash/execute_update_properties.sh +++ b/bash/execute_update_properties.sh @@ -1,12 +1,5 @@ #!/bin/bash -git clone https://github.com/citusdata/packaging.git -cd packaging -git checkout all-citus -python ../../python/update_package_properties.py --gh_token "${GITHUB_TOKEN}" --prj_name "${PRJ_NAME}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ - --fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S &z')" --exec-path "$(pwd)" &>2 +python ../../python/update_package_properties.py --gh_token "${GITHUB_TOKEN}" --prj_name "${PRJ_NAME}" --prj_ver "${PRJ_VER}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ +--fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S &z')" --exec-path "$(pwd)" &>2 -#cd .. -#echo "Removing Packaging directory..." -#rm -r packaging -#echo "Packaging directory Removed" diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 06636f91..2de2b02a 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -1,6 +1,6 @@ import argparse import re -from datetime import date +from datetime import date, datetime import pathlib2 import string_utils @@ -181,7 +181,6 @@ def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, tag_na print("Already version in the debian changelog") - @validate_parameters def update_pkgvars(version: is_version(non_empty(no_whitespaces(non_blank(str)))), fancy: bool, fancy_release_count: non_negative(int), templates_path: str, pkgvars_path: str) -> None: @@ -272,7 +271,8 @@ def update_rpm_spec(project_name: str, project_version: str, microsoft_email: st @validate_parameters -def update_all_changes(github_token: non_empty(non_blank(str)), project_name: non_empty(str), project_version: is_version(str), +def update_all_changes(github_token: non_empty(non_blank(str)), project_name: non_empty(str), + project_version: is_version(str), tag_name: is_tag(str), fancy: bool, fancy_version_number: non_negative(int), microsoft_email: is_email(str), name_surname: non_empty(non_blank(str)), release_date: date, packaging_path: str): @@ -303,5 +303,10 @@ def update_all_changes(github_token: non_empty(non_blank(str)), project_name: no parser.add_argument('--exec_path') args = parser.parse_args() - update_all_changes(args.gh_token, args.prj_name, args.tag_name, args.fancy, args.fancy_ver_no, args.email, - args.name, args.date, args.exec_path) + if not string_utils.is_integer(args.fancy_ver_no): + raise ValueError(f"fancy_ver_no is expected to be numeric actual value {args.fancy_ver_no}") + + exec_date = datetime.strptime(args.date, '%Y.%m.%d %H:%M:%S %z') + + update_all_changes(args.gh_token, args.prj_name, args.prj_ver, args.tag_name, args.fancy, int(args.fancy_ver_no), + args.email, args.name, exec_date, args.exec_path) From 72605a544dcd42cf69321956292f32a06aac296d Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 12:18:05 +0300 Subject: [PATCH 022/106] Fix bash command --- bash/execute_update_properties.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bash/execute_update_properties.sh b/bash/execute_update_properties.sh index 721d6447..995af6d9 100644 --- a/bash/execute_update_properties.sh +++ b/bash/execute_update_properties.sh @@ -1,5 +1,4 @@ #!/bin/bash - python ../../python/update_package_properties.py --gh_token "${GITHUB_TOKEN}" --prj_name "${PRJ_NAME}" --prj_ver "${PRJ_VER}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ --fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S &z')" --exec-path "$(pwd)" &>2 From f353723720e1ab8169c1f824f9d4adaedc15fdec Mon Sep 17 00:00:00 2001 From: Gurkan Indibay Date: Thu, 1 Apr 2021 11:54:34 +0000 Subject: [PATCH 023/106] Add update package properties --- bash/update_package_properties.sh | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 bash/update_package_properties.sh diff --git a/bash/update_package_properties.sh b/bash/update_package_properties.sh new file mode 100644 index 00000000..fbc1ea44 --- /dev/null +++ b/bash/update_package_properties.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +PRJ_NAME="citus" +PRJ_VER="10.0.3" +TAG_NAME="v${PRJ_VER}" +FANCY="True" +FANCY_VERSION_NO=2 +MICROSOFT_EMAIL="gindibay@gmail.com" +NAME="Gurkan Indibay" + +[ -z "${PRJ_NAME}" ] && echo "PRJ_NAME should be non-empty value" && exit 1 +[ -z "${PRJ_VER}" ] && echo "PRJ_VER should be non-empty value" && exit 1 +[ -z "${TAG_NAME}" ] && echo "TAG_NAME should be non-empty value" && exit 1 +[ -z "${FANCY}" ] && echo "FANCY should be non-empty value" && exit 1 +[ -z "${FANCY_VERSION_NO}" ] && echo "FANCY_VERSION_NO should be non-empty value" && exit 1 +[ -z "${MICROSOFT_EMAIL}" ] && echo "MICROSOFT_EMAIL should be non-empty value" && exit 1 +[ -z "${NAME}" ] && echo "NAME should be non-empty value" && exit 1 + +main_branch_name=$(git branch --show-current) + +pr_branch_name="${main_branch_name}-$(date +%s)" + +commit_message="Bump to ${PRJ_NAME} ${PRJ_VER}" + +git checkout -b "${pr_branch_name}" + +python tools/python/update_package_properties.py --gh_token "${GH_TOKEN}" --prj_name "${PRJ_NAME}" --prj_ver "${PRJ_VER}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ + --fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S %z')" --exec_path "$(pwd)" + +git commit -a -m "${commit_message}" + +echo "{\"title\":\"${commit_message}\", \"head\":\"${pr_branch_name}\", \"base\":\"${main_branch_name}\"}" + +git push origin "${pr_branch_name}" + +curl -g -H "Accept: application/vnd.github.v3.full+json" -X POST --user "${GH_TOKEN}:x-oauth-basic" -d "{\"title\":\"${commit_message}\", \"head\":\"${pr_branch_name}\", \"base\":\"${main_branch_name}\"}" https://api.github.com/repos/citusdata/packaging/pulls From 7e86b5d96f9f46ccb5050eac2893a451626b8141 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 16:23:42 +0300 Subject: [PATCH 024/106] Add flags to make script fail when a step fail --- bash/update_package_properties.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bash/update_package_properties.sh b/bash/update_package_properties.sh index fbc1ea44..4cb208aa 100644 --- a/bash/update_package_properties.sh +++ b/bash/update_package_properties.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -euxo pipefail + PRJ_NAME="citus" PRJ_VER="10.0.3" TAG_NAME="v${PRJ_VER}" From db00b6ae2830c5f8ae99d512f459d3705de368ed Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 1 Apr 2021 17:13:44 +0300 Subject: [PATCH 025/106] remove parameter set on update_package_properties.sh to get these parameters from pipeline parameters --- bash/update_package_properties.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bash/update_package_properties.sh b/bash/update_package_properties.sh index 4cb208aa..bbaeec51 100644 --- a/bash/update_package_properties.sh +++ b/bash/update_package_properties.sh @@ -2,13 +2,13 @@ set -euxo pipefail -PRJ_NAME="citus" -PRJ_VER="10.0.3" -TAG_NAME="v${PRJ_VER}" -FANCY="True" -FANCY_VERSION_NO=2 -MICROSOFT_EMAIL="gindibay@gmail.com" -NAME="Gurkan Indibay" +#PRJ_NAME="citus" +#PRJ_VER="10.0.3" +#TAG_NAME="v${PRJ_VER}" +#FANCY="True" +#FANCY_VERSION_NO=2 +#MICROSOFT_EMAIL="gindibay@gmail.com" +#NAME="Gurkan Indibay" [ -z "${PRJ_NAME}" ] && echo "PRJ_NAME should be non-empty value" && exit 1 [ -z "${PRJ_VER}" ] && echo "PRJ_VER should be non-empty value" && exit 1 From 03f4e8f88cc4d054f4294bffa1b60326c09e118f Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 2 Apr 2021 16:13:21 +0300 Subject: [PATCH 026/106] Fix some static code warnings Add checks to prevent Add checks and test cases to prevent multiple changelog add --- python/tests/files/citus.spec | 3 - python/tests/files/citus_include_10_0_3.spec | 315 +++ .../debian.changelog_include_10_0_3.refer | 1802 +++++++++++++++++ .../tests/test_update_package_properties.py | 34 +- python/update_package_properties.py | 39 +- 5 files changed, 2162 insertions(+), 31 deletions(-) create mode 100644 python/tests/files/citus_include_10_0_3.spec create mode 100644 python/tests/files/debian.changelog_include_10_0_3.refer diff --git a/python/tests/files/citus.spec b/python/tests/files/citus.spec index 6bbd61be..199fe73c 100644 --- a/python/tests/files/citus.spec +++ b/python/tests/files/citus.spec @@ -79,9 +79,6 @@ echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list %doc %{pginstdir}/doc/extension/README-%{sname}.md %changelog -* Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 -- Update to Citus 10.0.3 - * Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 - Official 10.0.2 release of Citus diff --git a/python/tests/files/citus_include_10_0_3.spec b/python/tests/files/citus_include_10_0_3.spec new file mode 100644 index 00000000..199fe73c --- /dev/null +++ b/python/tests/files/citus_include_10_0_3.spec @@ -0,0 +1,315 @@ +%global pgmajorversion 11 +%global pgpackageversion 11 +%global pginstdir /usr/pgsql-%{pgpackageversion} +%global sname citus + +Summary: PostgreSQL-based distributed RDBMS +Name: %{sname}%{?pkginfix}_%{pgmajorversion} +Provides: %{sname}_%{pgmajorversion} +Conflicts: %{sname}_%{pgmajorversion} +Version: 10.0.3-1 +Release: 1%{dist} +License: AGPLv3 +Group: Applications/Databases +Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz +URL: https://github.com/citusdata/ +BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel +Requires: postgresql%{pgmajorversion}-server +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +%description +Citus horizontally scales PostgreSQL across commodity servers +using sharding and replication. Its query engine parallelizes +incoming SQL queries across these servers to enable real-time +responses on large datasets. + +Citus extends the underlying database rather than forking it, +which gives developers and enterprises the power and familiarity +of a traditional relational database. As an extension, Citus +supports new PostgreSQL releases, allowing users to benefit from +new features while maintaining compatibility with existing +PostgreSQL tools. Note that Citus supports many (but not all) SQL +commands. + +%prep +%setup -q -n %{sname}-%{version} + +%build +%configure PG_CONFIG=%{pginstdir}/bin/pg_config --with-extra-version="%{?conf_extra_version}" --with-security-flags +make %{?_smp_mflags} + +%install +%make_install +# Install documentation with a better name: +%{__mkdir} -p %{buildroot}%{pginstdir}/doc/extension +%{__cp} README.md %{buildroot}%{pginstdir}/doc/extension/README-%{sname}.md +# Set paths to be packaged other than LICENSE, README & CHANGELOG.md +echo %{pginstdir}/include/server/citus_*.h >> installation_files.list +echo %{pginstdir}/include/server/distributed/*.h >> installation_files.list +echo %{pginstdir}/lib/%{sname}.so >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}-*.sql >> installation_files.list +echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list +%ifarch ppc64 ppc64le + %else + %if 0%{?rhel} && 0%{?rhel} <= 6 + %else + echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list + echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list + + # Columnar does not exist in Citus versions < 10.0 + # At this point, we don't have %{pginstdir}, + # so first check build directory for columnar. + [[ -d %{buildroot}%{pginstdir}/lib/bitcode/columnar/ ]] && echo %{pginstdir}/lib/bitcode/columnar/*.bc >> installation_files.list + %endif +%endif + +%clean +%{__rm} -rf %{buildroot} + +%files -f installation_files.list +%files +%defattr(-,root,root,-) +%doc CHANGELOG.md +%if 0%{?rhel} && 0%{?rhel} <= 6 +%doc LICENSE +%else +%license LICENSE +%endif +%doc %{pginstdir}/doc/extension/README-%{sname}.md + +%changelog +* Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 +- Official 10.0.2 release of Citus + +* Wed Jan 27 2021 - gurkanindibay 9.5.2.citus-1 +- Official 9.5.2 release of Citus + +* Tue Jan 5 2021 - gurkanindibay 9.4.4.citus-1 +- Official 9.4.4 release of Citus + +* Wed Dec 2 2020 - Onur Tirtir 9.5.1.citus-1 +- Official 9.5.1 release of Citus + +* Tue Nov 24 2020 - Onur Tirtir 9.4.3.citus-1 +- Official 9.4.3 release of Citus + +* Wed Nov 11 2020 - Onur Tirtir 9.5.0.citus-1 +- Official 9.5.0 release of Citus + +* Thu Oct 22 2020 - Onur Tirtir 9.4.2.citus-1 +- Official 9.4.2 release of Citus + +* Wed Sep 30 2020 - Onur Tirtir 9.4.1.citus-1 +- Official 9.4.1 release of Citus + +* Tue Jul 28 2020 - Onur Tirtir 9.4.0.citus-1 +- Official 9.4.0 release of Citus + +* Mon Jul 27 2020 - Onur Tirtir 9.3.5.citus-1 +- Official 9.3.5 release of Citus + +* Wed Jul 22 2020 - Onur Tirtir 9.3.4.citus-1 +- Official 9.3.4 release of Citus + +* Mon Jul 13 2020 - Onur Tirtir 9.3.3.citus-1 +- Official 9.3.3 release of Citus + +* Thu May 7 2020 - Onur Tirtir 9.3.0.citus-1 +- Update to Citus 9.3.0 + +* Tue Mar 31 2020 - Onur Tirtir 9.2.4.citus-1 +- Update to Citus 9.2.4 + +* Thu Mar 26 2020 - Onur Tirtir 9.2.3.citus-1 +- Update to Citus 9.2.3 + +* Fri Mar 6 2020 - Onur Tirtir 9.0.2.citus-1 +- Update to Citus 9.0.2 + +* Fri Mar 6 2020 - Onur Tirtir 9.2.2.citus-1 +- Update to Citus 9.2.2 + +* Fri Feb 14 2020 - Onur Tirtir 9.2.1.citus-1 +- Update to Citus 9.2.1 + +* Mon Feb 10 2020 - Onur Tirtir 9.2.0.citus-1 +- Update to Citus 9.2.0 + +* Wed Dec 18 2019 - Onur Tirtir 9.1.1.citus-1 +- Update to Citus 9.1.1 + +* Thu Nov 28 2019 - Onur Tirtir 9.1.0.citus-1 +- Update to Citus 9.1.0 + +* Wed Oct 30 2019 - Hanefi Onaldi 9.0.1.citus-1 +- Update to Citus 9.0.1 + +* Thu Oct 10 2019 - Hanefi Onaldi 9.0.0.citus-1 +- Update to Citus 9.0.0 + +* Fri Aug 9 2019 - Hanefi Onaldi 8.3.2.citus-1 +- Update to Citus 8.3.2 + +* Mon Jul 29 2019 - Hanefi Onaldi 8.3.1.citus-1 +- Update to Citus 8.3.1 + +* Wed Jul 10 2019 - Burak Velioglu 8.3.0.citus-1 +- Update to Citus 8.3.0 + +* Wed Jun 12 2019 - Burak Velioglu 8.2.2.citus-1 +- Update to Citus 8.2.2 + +* Wed Apr 3 2019 - Burak Velioglu 8.2.1.citus-1 +- Update to Citus 8.2.1 + +* Wed Apr 3 2019 - Burak Velioglu 8.1.2.citus-1 +- Update to Citus 8.1.2 + +* Thu Mar 28 2019 - Burak Velioglu 8.2.0.citus-1 +- Update to Citus 8.2.0 + +* Wed Jan 9 2019 - Burak Velioglu 8.0.3.citus-1 +- Update to Citus 8.0.3 + +* Mon Jan 7 2019 - Burak Velioglu 8.1.1.citus-1 +- Update to Citus 8.1.1 + +* Tue Dec 18 2018 - Burak Velioglu 8.1.0.citus-1 +- Update to Citus 8.1.0 + +* Thu Dec 13 2018 - Burak Velioglu 8.0.2.citus-1 +- Update to Citus 8.0.2 + +* Wed Dec 12 2018 - Burak Velioglu 7.5.4.citus-1 +- Update to Citus 7.5.4 + +* Wed Nov 28 2018 - Burak Velioglu 8.0.1.citus-1 +- Update to Citus 8.0.1 + +* Wed Nov 28 2018 - Burak Velioglu 7.5.3.citus-1 +- Update to Citus 7.5.3 + +* Wed Nov 14 2018 - Burak Velioglu 7.5.2.citus-1 +- Update to Citus 7.5.2 + +* Fri Nov 02 2018 - Burak Velioglu 8.0.0.citus-1 +- Update to Citus 8.0.0 + +* Wed Aug 29 2018 - Burak Velioglu 7.5.1.citus-1 +- Update to Citus 7.5.1 + +* Fri Jul 27 2018 - Mehmet Furkan Sahin 7.4.2.citus-1 +- Update to Citus 7.4.2 + +* Wed Jul 25 2018 - Mehmet Furkan Sahin 7.5.0.citus-1 +- Update to Citus 7.5.0 + +* Wed Jun 20 2018 - Burak Velioglu 7.4.1.citus-1 +- Update to Citus 7.4.1 + +* Thu May 17 2018 - Burak Velioglu 7.2.2.citus-1 +- Update to Citus 7.2.2 + +* Tue May 15 2018 - Burak Velioglu 7.4.0.citus-1 +- Update to Citus 7.4.0 + +* Thu Mar 15 2018 - Burak Velioglu 7.3.0.citus-1 +- Update to Citus 7.3.0 + +* Tue Feb 6 2018 - Burak Velioglu 7.2.1.citus-1 +- Update to Citus 7.2.1 + +* Tue Jan 16 2018 - Burak Velioglu 7.2.0.citus-1 +- Update to Citus 7.2.0 + +* Thu Jan 11 2018 - Burak Velioglu 6.2.5.citus-1 +- Update to Citus 6.2.5 + +* Fri Jan 05 2018 - Burak Velioglu 7.1.2.citus-1 +- Update to Citus 7.1.2 + +* Tue Dec 05 2017 - Burak Velioglu 7.1.1.citus-1 +- Update to Citus 7.1.1 + +* Wed Nov 15 2017 - Burak Velioglu 7.1.0.citus-1 +- Update to Citus 7.1.0 + +* Mon Oct 16 2017 - Burak Yucesoy 7.0.3.citus-1 +- Update to Citus 7.0.3 + +* Thu Sep 28 2017 - Burak Yucesoy 7.0.2.citus-1 +- Update to Citus 7.0.2 + +* Thu Sep 28 2017 - Burak Yucesoy 6.2.4.citus-1 +- Update to Citus 6.2.4 + +* Thu Sep 28 2017 - Burak Yucesoy 6.1.3.citus-1 +- Update to Citus 6.1.3 + +* Tue Sep 12 2017 - Burak Yucesoy 7.0.1.citus-1 +- Update to Citus 7.0.1 + +* Tue Aug 29 2017 - Burak Yucesoy 7.0.0.citus-1 +- Update to Citus 7.0.0 + +* Thu Jul 13 2017 - Burak Yucesoy 6.2.3.citus-1 +- Update to Citus 6.2.3 + +* Wed Jun 7 2017 - Burak Velioglu 6.2.2.citus-1 +- Update to Citus 6.2.2 + +* Wed Jun 7 2017 - Jason Petersen 6.1.2.citus-1 +- Update to Citus 6.1.2 + +* Wed May 24 2017 - Jason Petersen 6.2.1.citus-1 +- Update to Citus 6.2.1 + +* Tue May 16 2017 - Burak Yucesoy 6.2.0.citus-1 +- Update to Citus 6.2.0 + +* Fri May 5 2017 - Metin Doslu 6.1.1.citus-1 +- Update to Citus 6.1.1 + +* Thu Feb 9 2017 - Burak Yucesoy 6.1.0.citus-1 +- Update to Citus 6.1.0 + +* Wed Feb 8 2017 - Jason Petersen 6.0.1.citus-2 +- Transitional package to guide users to new package name + +* Wed Nov 30 2016 - Burak Yucesoy 6.0.1.citus-1 +- Update to Citus 6.0.1 + +* Tue Nov 8 2016 - Jason Petersen 6.0.0.citus-1 +- Update to Citus 6.0.0 + +* Tue Nov 8 2016 - Jason Petersen 5.2.2.citus-1 +- Update to Citus 5.2.2 + +* Tue Sep 6 2016 - Jason Petersen 5.2.1.citus-1 +- Update to Citus 5.2.1 + +* Wed Aug 17 2016 - Jason Petersen 5.2.0.citus-1 +- Update to Citus 5.2.0 + +* Mon Aug 1 2016 - Jason Petersen 5.2.0-0.1.rc.1 +- Release candidate for 5.2 + +* Fri Jun 17 2016 - Jason Petersen 5.1.1-1 +- Update to Citus 5.1.1 + +* Tue May 17 2016 - Jason Petersen 5.1.0-1 +- Update to Citus 5.1.0 + +* Mon May 16 2016 - Jason Petersen 5.1.0-0.2.rc.2 +- Fix EXPLAIN output when FORMAT JSON in use + +* Wed May 4 2016 - Jason Petersen 5.1.0-0.1.rc.1 +- Release candidate for 5.1 + +* Fri Apr 15 2016 - Jason Petersen 5.0.1-1 +- Fixes issues on 32-bit systems + +* Fri Mar 25 2016 - Devrim Gündüz 5.0.0-1 +- Initial RPM packaging for PostgreSQL RPM Repository, + based on the spec file of Jason Petersen @ Citus. diff --git a/python/tests/files/debian.changelog_include_10_0_3.refer b/python/tests/files/debian.changelog_include_10_0_3.refer new file mode 100644 index 00000000..d6eaae12 --- /dev/null +++ b/python/tests/files/debian.changelog_include_10_0_3.refer @@ -0,0 +1,1802 @@ +citus (10.0.3.citus-1) stable; urgency=low + + * Prevents infinite recursion for queries that involve UNION ALL + below `JOIN` + + * Fixes a crash in queries with a modifying CTE and a SELECT + without `FROM` + + * Fixes upgrade and downgrade paths for citus_update_table_statistics + + * Fixes a bug that causes SELECT queries to use 2PC unnecessarily + + * Fixes a bug that might cause self-deadlocks with + `CREATE INDEX` / `REINDEX CONCURRENTLY` commands + + * Adds citus.max_cached_connection_lifetime GUC to set maximum connection + lifetime + + * Adds citus.remote_copy_flush_threshold GUC that controls + per-shard memory usages by `COPY` + + * Adds citus_get_active_worker_nodes UDF to deprecate + `master_get_active_worker_nodes` + + * Skips 2PC for readonly connections in a transaction + + * Makes sure that local execution starts coordinated transaction + + * Removes open temporary file warning when cancelling a query with + an open tuple store + + * Relaxes the locks when adding an existing node + + -- Gurkan Indibay Thu, 18 Mar 2021 01:40:08 +0000 + +citus (10.0.2.citus-1) stable; urgency=low + + * Adds a configure flag to enforce security + + * Fixes a bug due to cross join without target list + + * Fixes a bug with UNION ALL on PG 13 + + * Fixes a compatibility issue with pg_audit in utility calls + + * Fixes insert query with CTEs/sublinks/subqueries etc + + * Grants SELECT permission on citus_tables view to public + + * Grants SELECT permission on columnar metadata tables to public + + * Improves citus_update_table_statistics and provides distributed deadlock + detection + + * Preserves colocation with procedures in alter_distributed_table + + * Prevents using alter_columnar_table_set and alter_columnar_table_reset + on a columnar table not owned by the user + + * Removes limits around long table names + + -- Gurkan Indibay Thu, 4 Mar 2021 2:46:54 +0000 + +citus (9.5.2.citus-1) stable; urgency=low + + * Fixes distributed deadlock detection being blocked by metadata sync + + * Prevents segfaults when SAVEPOINT handling cannot recover from connection + failures + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Wed, 27 Jan 2021 11:25:38 +0000 + +citus (9.4.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause router queries with local tables to be pushed + down + + * Fixes a segfault in connection management due to invalid connection hash + entries + + * Fixes possible issues that might occur with single shard distributed tables + + -- gurkanindibay Tue, 5 Jan 2021 14:58:56 +0000 + +citus (9.5.1.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that could cause excessive memory consumption when a partition is + created + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + * Fixes a segfault in connection management due to invalid connection hash + entries + + -- Onur Tirtir Wed, 2 Dec 2020 14:28:44 +0000 + +citus (9.4.3.citus-1) stable; urgency=low + + * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE + + * Fixes a bug that triggers subplan executions unnecessarily with cursors + + -- Onur Tirtir Tue, 24 Nov 2020 11:17:57 +0000 + +citus (9.5.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 13 + + * Removes the task-tracker executor + + * Introduces citus local tables + + * Introduces undistribute_table UDF to convert tables back to postgres tables + + * Adds support for EXPLAIN (ANALYZE) EXECUTE and EXPLAIN EXECUTE + + * Adds support for EXPLAIN (ANALYZE, WAL) for PG13 + + * Sorts the output of EXPLAIN (ANALYZE) by execution duration. + + * Adds support for CREATE TABLE ... USING table_access_method + + * Adds support for WITH TIES option in SELECT and INSERT SELECT queries + + * Avoids taking multi-shard locks on workers + + * Enforces citus.max_shared_pool_size config in COPY queries + + * Enables custom aggregates with multiple parameters to be executed on workers + + * Enforces citus.max_intermediate_result_size in local execution + + * Improves cost estimation of INSERT SELECT plans + + * Introduces delegation of procedures that read from reference tables + + * Prevents pull-push execution for simple pushdownable subqueries + + * Improves error message when creating a foreign key to a local table + + * Makes citus_prepare_pg_upgrade idempotent by dropping transition tables + + * Disallows ON TRUE outer joins with reference & distributed tables when + reference table is outer relation to avoid incorrect results + + * Disallows field indirection in INSERT/UPDATE queries to avoid incorrect + results + + * Disallows volatile functions in UPDATE subqueries to avoid incorrect results + + * Fixes CREATE INDEX CONCURRENTLY crash with local execution + + * Fixes citus_finish_pg_upgrade to drop all backup tables + + * Fixes a bug that cause failures when RECURSIVE VIEW joined reference table + + * Fixes DROP SEQUENCE failures when metadata syncing is enabled + + * Fixes a bug that caused CREATE TABLE with CHECK constraint to fail + + * Fixes a bug that could cause VACUUM to deadlock + + * Fixes master_update_node failure when no background worker slots are available + + * Fixes a bug that caused replica identity to not be propagated on shard repair + + * Fixes a bug that could cause crashes after connection timeouts + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug that could cause deadlocks on CREATE INDEX + + * Fixes a bug with genetic query optimization in outer joins + + * Fixes a crash when aggregating empty tables + + * Fixes a crash with inserting domain constrained composite types + + * Fixes a crash with multi-row & router INSERT's in local execution + + * Fixes a possibility of doing temporary file cleanup more than once + + * Fixes incorrect setting of join related fields + + * Fixes memory issues around deparsing index commands + + * Fixes reference table access tracking for sequential execution + + * Fixes removal of a single node with only reference tables + + * Fixes sending commands to coordinator when it is added as a worker + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlock + + -- Onur Tirtir Wed, 11 Nov 2020 15:00:27 +0000 + +citus (9.4.2.citus-1) stable; urgency=low + + * Fixes a bug that could lead to multiple maintenance daemons + + * Fixes an issue preventing views in reference table modifications + + -- Onur Tirtir Thu, 22 Oct 2020 8:53:44 +0000 + +citus (9.4.1.citus-1) stable; urgency=low + + * Fixes EXPLAIN ANALYZE output truncation + + * Fixes a deadlock during transaction recovery + + -- Onur Tirtir Wed, 30 Sep 2020 9:33:46 +0000 + +citus (9.4.0.citus-1) stable; urgency=low + + * Improves COPY by honoring max_adaptive_executor_pool_size config + + * Adds support for insert into local table select from distributed table + + * Adds support to partially push down tdigest aggregates + + * Adds support for receiving binary encoded results from workers using + citus.enable_binary_protocol + + * Enables joins between local tables and CTEs + + * Adds showing query text in EXPLAIN output when explain verbose is true + + * Adds support for showing CTE statistics in EXPLAIN ANALYZE + + * Adds support for showing amount of data received in EXPLAIN ANALYZE + + * Introduces downgrade paths in migration scripts + + * Avoids returning incorrect results when changing roles in a transaction + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes a bug that occurs when coordinator is added as a worker node + + * Fixes a crash because of overflow in partition id with certain compile flags + + * Fixes a crash that may happen if no worker nodes are added + + * Fixes a crash that occurs when inserting implicitly coerced constants + + * Fixes a crash when aggregating empty tables + + * Fixes a memory leak in subtransaction memory handling + + * Fixes crash when using rollback to savepoint after cancellation of DML + + * Fixes deparsing for queries with anonymous column references + + * Fixes distribution of composite types failing to include typemods + + * Fixes explain analyze on adaptive executor repartitions + + * Fixes possible error throwing in abort handle + + * Fixes segfault when evaluating func calls with default params on coordinator + + * Fixes several EXPLAIN ANALYZE issues + + * Fixes write queries with const expressions and COLLATE in various places + + * Fixes wrong cancellation message about distributed deadlocks + + * Reports correct INSERT/SELECT method in EXPLAIN + + * Disallows triggers on citus tables + + -- Onur Tirtir Tue, 28 Jul 2020 13:22:31 +0000 + +citus (9.3.5.citus-1) stable; urgency=low + + * Fixes ALTER TABLE IF EXISTS SET SCHEMA with non-existing table bug + + * Fixes CREATE INDEX CONCURRENTLY with no index name on a postgres table bug + + * Fixes a crash because of overflow in partition id with certain compile flags + + -- Onur Tirtir Mon, 27 Jul 2020 7:28:18 +0000 + +citus (9.3.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes with certain compile flags + + * Fixes a bug with lists of configuration values in ALTER ROLE SET statements + + * Fixes deparsing for queries with anonymous column references + + -- Onur Tirtir Wed, 22 Jul 2020 9:00:01 +0000 + +citus (9.3.3.citus-1) stable; urgency=low + + * Fixes a memory leak in subtransaction memory handling + + -- Onur Tirtir Mon, 13 Jul 2020 8:47:40 +0000 + +citus (9.3.0.citus-1) stable; urgency=low + + * Adds max_shared_pool_size to control number of connections across sessions + + * Adds support for window functions on coordinator + + * Improves shard pruning logic to understand OR-conditions + + * Prevents using an extra connection for intermediate result multi-casts + + * Adds propagation of ALTER ROLE .. SET statements + + * Adds update_distributed_table_colocation UDF to update colocation of tables + + * Introduces a UDF to truncate local data after distributing a table + + * Adds support for creating temp schemas in parallel + + * Adds support for evaluation of nextval in the target list on coordinator + + * Adds support for local execution of COPY/TRUNCATE/DROP/DDL commands + + * Adds support for local execution of shard creation + + * Uses local execution in a transaction block + + * Adds support for querying distributed table sizes concurrently + + * Allows master_copy_shard_placement to replicate placements to new nodes + + * Allows table type to be used in target list + + * Avoids having multiple maintenance daemons active for a single database + + * Defers reference table replication to shard creation time + + * Enables joins between local tables and reference tables in transaction blocks + + * Ignores pruned target list entries in coordinator plan + + * Improves SIGTERM handling of maintenance daemon + + * Increases the default of citus.node_connection_timeout to 30 seconds + + * Fixes a bug that occurs when creating remote tasks in local execution + + * Fixes a bug that causes some DML queries containing aggregates to fail + + * Fixes a bug that could cause failures in queries with subqueries or CTEs + + * Fixes a bug that may cause some connection failures to throw errors + + * Fixes a bug which caused queries with SRFs and function evalution to fail + + * Fixes a bug with generated columns when executing COPY dist_table TO file + + * Fixes a crash when using non-constant limit clauses + + * Fixes a failure when composite types used in prepared statements + + * Fixes a possible segfault when dropping dist. table in a transaction block + + * Fixes a possible segfault when non-pushdownable aggs are solely used in HAVING + + * Fixes a segfault when executing queries using GROUPING + + * Fixes an error when using LEFT JOIN with GROUP BY on primary key + + * Fixes an issue with distributing tables having generated cols not at the end + + * Fixes automatic SSL permission issue when using "initdb --allow-group-access" + + * Fixes errors which could occur when subqueries are parameters to aggregates + + * Fixes possible issues by invalidating the plan cache in master_update_node + + * Fixes timing issues which could be caused by changing system clock + + -- Onur Tirtir Thu, 7 May 2020 15:11:25 +0000 + +citus (9.2.4.citus-1) stable; urgency=low + + * Fixes a release problem in 9.2.3 + + -- Onur Tirtir Tue, 31 Mar 2020 08:06:59 +0000 + +citus (9.2.3.citus-1) stable; urgency=low + + * Do not use C functions that have been banned by Microsoft + + * Fixes a bug that causes wrong results with complex outer joins + + * Fixes issues found using static analysis + + * Fixes left join shard pruning in pushdown planner + + * Fixes possibility for segmentation fault in internal aggregate functions + + * Fixes possible segfault when non pushdownable aggregates are used in HAVING + + * Improves correctness of planning subqueries in HAVING + + * Prevents using old connections for security if citus.node_conninfo changed + + * Uses Microsoft approved cipher string for default TLS setup + + -- Onur Tirtir Thu, 26 Mar 2020 8:22:48 +0000 + +citus (9.0.2.citus-1) stable; urgency=low + + * Fixes build errors on EL/OL 6 based distros + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Disallows distributed function creation when replication_model is `statement` + + -- Onur Tirtir Fri, 6 Mar 2020 14:10:16 +0000 + +citus (9.2.2.citus-1) stable; urgency=low + + * Fixes a bug that caused some prepared stmts with function calls to fail + + * Fixes a bug that caused some prepared stmts with composite types to fail + + * Fixes a bug that caused missing subplan results in workers + + * Improves performance of re-partition joins + + -- Onur Tirtir Fri, 6 Mar 2020 07:14:20 +0000 + +citus (9.2.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause crashes if distribution key is NULL + + -- Onur Tirtir Fri, 14 Feb 2020 11:51:09 +0000 + +citus (9.2.0.citus-1) stable; urgency=low + + * Adds support for INSERT...SELECT queries with re-partitioning + + * Adds citus.coordinator_aggregation_strategy to support more aggregates + + * Adds caching of local plans on shards for Citus MX + + * Adds compatibility support for dist. object infrastructure from old versions + + * Adds defering shard-pruning for fast-path router queries to execution + + * Adds propagation of GRANT ... ON SCHEMA queries + + * Adds support for CTE pushdown via CTE inlining in distributed planning + + * Adds support for ALTER TABLE ... SET SCHEMA propagation. + + * Adds support for DROP ROUTINE & ALTER ROUTINE commands + + * Adds support for any inner join on a reference table + + * Changes citus.log_remote_commands level to NOTICE + + * Disallows marking ref. table shards unhealthy in the presence of savepoints + + * Disallows placing new shards with shards in TO_DELETE state + + * Enables local execution of queries that do not need any data access + + * Fixes Makefile trying to cleanup PG directory during install + + * Fixes a bug causing errors when planning a query with multiple subqueries + + * Fixes a possible deadlock that could happen during shard moves + + * Fixes a problem when adding a new node due to tables referenced in func body + + * Fixes an issue that could cause joins with reference tables to be slow + + * Fixes cached metadata for shard is inconsistent issue + + * Fixes inserting multiple composite types as partition key in VALUES + + * Fixes unnecessary repartition on joins with more than 4 tables + + * Prevents wrong results for replicated partitioned tables after failure + + * Restricts LIMIT approximation for non-commutative aggregates + + -- Onur Tirtir Wed, 10 Feb 2020 8:48:00 +0000 + +citus (9.1.1.citus-1) stable; urgency=low + + * Fixes a bug causing SQL-executing UDFs to crash when passing in DDL + + * Fixes a bug that caused column_to_column_name to crash for invalid input + + * Fixes a bug that caused inserts into local tables w/ dist. subqueries to crash + + * Fixes a bug that caused some noop DML statements to fail + + * Fixes a bug that prevents dropping reference table columns + + * Fixes a crash in IN (.., NULL) queries + + * Fixes a crash when calling a distributed function from PL/pgSQL + + * Fixes an issue that caused CTEs to sometimes leak connections + + * Fixes strange errors in DML with unreachable sublinks + + * Prevents statements in SQL functions to run outside of a transaction + + -- Onur Tirtir Wed, 18 Dec 2019 14:32:42 +0000 + +citus (9.1.0.citus-1) stable; urgency=low + + * Adds extensions to distributed object propagation infrastructure + + * Adds support for ALTER ROLE propagation + + * Adds support for aggregates in create_distributed_function + + * Adds support for expressions in reference joins + + * Adds support for returning RECORD in multi-shard queries + + * Adds support for simple IN subqueries on unique cols in repartition joins + + * Adds support for subqueries in HAVING clauses + + * Automatically distributes unary aggs w/ combinefunc and non-internal stype + + * Disallows distributed func creation when replication_model is 'statement' + + * Drops support for deprecated real-time and router executors + + * Fixes a bug in local execution that could cause missing rows in RETURNING + + * Fixes a bug that caused maintenance daemon to fail on standby nodes + + * Fixes a bug that caused other CREATE EXTENSION commands to take longer + + * Fixes a bug that prevented REFRESH MATERIALIZED VIEW + + * Fixes a bug when view is used in modify statements + + * Fixes a memory leak in adaptive executor when query returns many columns + + * Fixes underflow init of default values in worker extended op node creation + + * Fixes potential segfault in standard_planner inlining functions + + * Fixes an issue that caused failures in RHEL 6 builds + + * Fixes queries with repartition joins and group by unique column + + * Improves CTE/Subquery performance by pruning intermediate rslt broadcasting + + * Removes citus.worker_list_file GUC + + * Revokes usage from the citus schema from public + + -- Onur Tirtir Thu, 28 Nov 2019 15:11:05 +0000 + +citus (9.0.1.citus-1) stable; urgency=low + + * Fixes a memory leak in the executor + + * Revokes usage from the citus schema from public + + -- Hanefi Onaldi Wed, 30 Oct 2019 8:53:22 +0000 + +citus (9.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 12 + + * Adds UDFs to help with PostgreSQL upgrades + + * Distributes types to worker nodes + + * Introduces create_distributed_function UDF + + * Introduces local query execution for Citus MX + + * Implements infrastructure for routing CALL to MX workers + + * Implements infrastructure for routing SELECT function() to MX workers + + * Adds support for foreign key constraints between reference tables + + * Adds a feature flag to turn off CREATE TYPE propagation + + * Adds option citus.single_shard_commit_protocol + + * Adds support for EXPLAIN SUMMARY + + * Adds support for GENERATE ALWAYS AS STORED + + * Adds support for serial and smallserial in MX mode + + * Adds support for anon composite types on the target list in router queries + + * Avoids race condition between create_reference_table & master_add_node + + * Fixes a bug in schemas of distributed sequence definitions + + * Fixes a bug that caused run_command_on_colocated_placements to fail + + * Fixes a bug that leads to various issues when a connection is lost + + * Fixes a schema leak on CREATE INDEX statement + + * Fixes assert failure in bare SELECT FROM reference table FOR UPDATE in MX + + * Makes master_update_node MX compatible + + * Prevents pg_dist_colocation from multiple records for reference tables + + * Prevents segfault in worker_partition_protocol edgecase + + * Propagates ALTER FUNCTION statements for distributed functions + + * Propagates CREATE OR REPLACE FUNCTION for distributed functions + + * Propagates REINDEX on tables & indexes + + * Provides a GUC to turn of the new dependency propagation functionality + + * Uses 2PC in adaptive executor when dealing with replication factors above 1 + + -- Hanefi Onaldi Tue, 15 Oct 2019 16:54:50 +0000 + +citus (8.3.2.citus-1) stable; urgency=low + + * Fixes performance issues by skipping unnecessary relation access recordings + + -- Hanefi Onaldi Fri, 9 Aug 2019 11:15:57 +0000 + +citus (8.3.1.citus-1) stable; urgency=low + + * Improves Adaptive Executor performance + + -- Hanefi Onaldi Mon, 29 Jul 2019 10:25:50 +0000 + +citus (8.3.0.citus-1) stable; urgency=low + + * Adds a new distributed executor: Adaptive Executor + + * citus.enable_statistics_collection defaults to off (opt-in) + + * Adds support for CTEs in router planner for modification queries + + * Adds support for propagating SET LOCAL at xact start + + * Adds option to force master_update_node during failover + + * Deprecates master_modify_multiple_shards + + * Improves round robin logic on router queries + + * Creates all distributed schemas as superuser on a separate connection + + * Makes COPY adapt to connection use behaviour of previous commands + + * Replaces SESSION_LIFESPAN with configurable no. of connections at xact end + + * Propagates ALTER FOREIGN TABLE commands to workers + + * Don't schedule tasks on inactive nodes + + * Makes DROP/VALIDATE CONSTRAINT tolerant of ambiguous shard extension + + * Fixes an issue with subquery map merge jobs as non-root + + * Fixes null pointers caused by partial initialization of ConnParamsHashEntry + + * Fixes errors caused by joins with shadowed aliases + + * Fixes a regression in outer joining subqueries introduced in 8.2.0 + + * Fixes a crash that can occur under high memory load + + * Fixes a bug that selects wrong worker when using round-robin assignment + + * Fixes savepoint rollback after multi-shard modify/copy failure + + * Fixes bad foreign constraint name search + + * Fixes a bug that prevents stack size to be adjusted + + -- Hanefi Onaldi Wed, 10 Jul 2019 15:19:02 +0000 + +citus (8.2.2.citus-1) stable; urgency=low + + * Fixes a bug in outer joins wrapped in subqueries + + -- Burak Velioglu Wed, 12 Jun 2019 8:45:08 +0000 + +citus (8.2.1.citus-1) stable; urgency=low + + * Fixes a bug that prevents stack size to be adjusted + + -- Burak Velioglu Wed, 3 Apr 2019 20:56:47 +0000 + +citus (8.1.2.citus-1) stable; urgency=low + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Fixes a bug that prevents stack size to be adjusted + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + -- Burak Velioglu Wed, 3 Apr 2019 20:34:46 +0000 + +citus (8.2.0.citus-1) stable; urgency=low + + * Removes support and code for PostgreSQL 9.6 + + * Enable more outer joins with reference tables + + * Execute CREATE INDEX CONCURRENTLY in parallel + + * Treat functions as transaction blocks + + * Add support for column aliases on join clauses + + * Skip standard_planner() for trivial queries + + * Added support for function calls in joins + + * Round-robin task assignment policy relies on local transaction id + + * Relax subquery union pushdown restrictions for reference tables + + * Speed-up run_command_on_shards() + + * Address some memory issues in connection config + + * Restrict visibility of get_*_active_transactions functions to pg_monitor + + * Don't do redundant ALTER TABLE consistency checks at coordinator + + * Queries with only intermediate results do not rely on task assignment policy + + * Finish connection establishment in parallel for multiple connections + + * Fixes a bug related to pruning shards using a coerced value + + * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands + + * Fixes a bug that could lead to infinite recursion during recursive planning + + * Fixes a bug that could prevent planning full outer joins with using clause + + * Fixes a bug that could lead to memory leak on citus_relation_size + + * Fixes a problem that could cause segmentation fault with recursive planning + + * Switch CI solution to CircleCI + + -- Burak Velioglu Fri, 29 Mar 2019 07:36:09 +0000 + +citus (8.0.3.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Wed, 9 Jan 2019 9:50:07 +0000 + +citus (8.1.1.citus-1) stable; urgency=low + + * Fixes maintenance daemon panic due to unreleased spinlock + + * Fixes an issue with having clause when used with complex joins + + -- Hanefi Onaldi Mon, 7 Jan 2019 16:26:13 +0000 + +citus (8.1.0.citus-1) stable; urgency=low + + * Turns on ssl by default for new installations of citus + + * Restricts SSL Ciphers to TLS1.2 and above + + * Adds support for INSERT INTO SELECT..ON CONFLICT/RETURNING via coordinator + + * Adds support for round-robin task assignment for queries to reference tables + + * Adds support for SQL tasks using worker_execute_sql_task UDF with task-tracker + + * Adds support for VALIDATE CONSTRAINT queries + + * Adds support for disabling hash aggregate with HLL + + * Adds user ID suffix to intermediate files generated by task-tracker + + * Only allow transmit from pgsql_job_cache directory + + * Disallows GROUPING SET clauses in subqueries + + * Removes restriction on user-defined group ID in node addition functions + + * Relaxes multi-shard modify locks when enable_deadlock_prevention is disabled + + * Improves security in task-tracker protocol + + * Improves permission checks in internal DROP TABLE functions + + * Improves permission checks in cluster management functions + + * Cleans up UDFs and fixes permission checks + + * Fixes crashes caused by stack size increase under high memory load + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Tue, 18 Dec 2018 15:12:45 +0000 + +citus (8.0.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Thu, 13 Dec 2018 13:56:44 +0000 + +citus (7.5.4.citus-1) stable; urgency=low + + * Fixes a bug that could cause maintenance daemon panic + + -- Burak Velioglu Wed, 12 Dec 2018 11:45:24 +0000 + +citus (8.0.1.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 11:38:47 +0000 + +citus (7.5.3.citus-1) stable; urgency=low + + * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker + + -- Burak Velioglu Wed, 28 Nov 2018 10:52:20 +0000 + +citus (7.5.2.citus-1) stable; urgency=low + + * Fixes inconsistent metadata error when shard metadata caching get interrupted + + * Fixes a bug that could cause memory leak + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes crashes caused by stack size increase under high memory load + + -- Burak Velioglu Wed, 14 Nov 2018 20:42:16 +0000 + +citus (8.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 11 + + * Adds support for applying DML operations on reference tables from MX nodes + + * Adds distributed locking to truncated MX tables + + * Adds support for running TRUNCATE command from MX worker nodes + + * Adds views to provide insight about the distributed transactions + + * Adds support for TABLESAMPLE in router queries + + * Adds support for INCLUDE option in index creation + + * Adds option to allow simple DML commands from hot standby + + * Adds support for partitioned tables with replication factor > 1 + + * Prevents a deadlock on concurrent DROP TABLE and SELECT on Citus MX + + * Fixes a bug that prevents recovering wrong transactions in MX + + * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load + + * Fixes a bug in MX mode, calling DROP SCHEMA with existing partitioned table + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + * Fixes a bug preventing rollback in CREATE PROCEDURE + + * Fixes a bug on not being able to drop index on a partitioned table + + * Fixes a bug on TRUNCATE when there is a foreign key to a reference table + + * Fixes a performance issue in prepared INSERT..SELECT + + * Fixes a bug which causes errors on DROP DATABASE IF EXISTS + + * Fixes a bug to remove intermediate result directory in pull-push execution + + * Improves query pushdown planning performance + + * Evaluate functions anywhere in query + + -- Burak Velioglu Fri, 02 Nov 2018 08:06:42 +0000 + +citus (7.5.1.citus-1) stable; urgency=low + + * Improves query pushdown planning performance + + * Fixes a bug that could cause modifying CTEs to select wrong execution mode + + -- Burak Velioglu Wed, 29 Aug 2018 08:06:42 +0000 + +citus (7.4.2.citus-1) stable; urgency=low + + * Fixes a segfault in real-time executor during online shard move + + -- Mehmet Furkan Sahin Fri, 27 Jul 2018 13:42:27 +0000 + +citus (7.5.0.citus-1) stable; urgency=low + + * Adds foreign key support from hash distributed to reference tables + + * Adds SELECT ... FOR UPDATE support for router plannable queries + + * Adds support for non-partition columns in count distinct + + * Fixes a segfault in real-time executor during online shard move + + * Fixes ALTER TABLE ADD COLUMN constraint check + + * Fixes a bug where INSERT ... SELECT allows one to update dist. column + + * Allows DDL commands to be sequentialized via citus.multi_shard_modify_mode + + * Adds support for topn_union_agg and topn_add_agg across shards + + * Adds support for hll_union_agg and hll_add_agg across shards + + * Fixes a bug that might cause shards to have a wrong owner + + * Adds select_opens_transaction_block GUC + + * Adds utils to implement DDLs for policies in future + + * Makes intermediate results to use separate connections + + * Adds a node_conninfo GUC to set outgoing connection settings + + -- Mehmet Furkan Sahin Wed, 25 Jul 2018 9:32:24 +0000 + +citus (6.2.6.citus-1) stable; urgency=low + + * Adds support for respecting enable_hashagg in the master planner + + -- Burak Velioglu Fri, 06 Jul 2018 13:30:08 +0000 + +citus (7.4.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause txns to incorrectly proceed after failure + + * Fixes a bug on INSERT ... SELECT queries in prepared statements + + -- Burak Velioglu Wed, 20 Jun 2018 12:25:30 +0000 + +citus (7.2.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause SELECTs to crash during a rebalance + + -- Burak Velioglu Thu, 17 May 2018 11:51:56 +0000 + +citus (7.4.0.citus-1) stable; urgency=low + + * Adds support for non-pushdownable subqueries and CTEs in UPDATE/DELETE + + * Adds support for pushdownable subqueries and joins in UPDATE/DELETE + + * Adds faster shard pruning for subqueries + + * Adds partitioning support to MX table + + * Adds support for (VACUUM | ANALYZE) VERBOSE + + * Adds support for multiple ANDs in HAVING for pushdown planner + + * Adds support for quotation needy schema names + + * Improves operator check time in physical planner for custom data types + + * Removes broadcast join logic + + * Deprecates large_table_shard_count and master_expire_table_cache() + + * Modifies master_update_node to write-lock shards hosted by node over update + + * DROP TABLE now drops shards as the currrent user instead of the superuser + + * Adds specialised error codes for connection failures + + * Improves error messages on connection failure + + * Fixes issue which prevented multiple citus_table_size calls per query + + * Tests are updated to use create_distributed_table + + -- Burak Velioglu Tue, 15 May 2018 13:01:17 +0000 + +citus (7.3.0.citus-1) stable; urgency=low + + * Adds support for non-colocated joins between subqueries + + * Adds support for window functions that can be pushed down to worker + + * Adds support for modifying CTEs + + * Adds recursive plan for WHERE clause subqueries with recurring FROM clause + + * Adds support for bool_ and bit_ aggregates + + * Adds support for Postgres jsonb and json aggregation functions + + * Adds support for respecting enable_hashagg in the master plan + + * Performance improvements to reduce distributed planning time + + * Fixes a bug on planner when aggregate is used in ORDER BY + + * Fixes a bug on planner when DISTINCT (ON) clause is used with GROUP BY + + * Fixes a planner bug with distinct and aggregate clauses + + * Fixes a bug that opened new connections on each table size function call + + * Fixes a bug canceling backends not involved in distributed deadlocks + + * Fixes count distinct bug on column expressions when used with subqueries + + * Improves error handling on worker node failures + + * Improves error messages for INSERT queries that have subqueries + + -- Burak Velioglu Thu, 15 Mar 2018 14:16:10 +0000 + +citus (7.2.1.citus-1) stable; urgency=low + + * Fixes count distinct bug on column expressions when used with subqueries + + * Adds support for respecting enable_hashagg in the master plan + + * Fixes a bug canceling backends not involved in distributed deadlocks + + -- Burak Velioglu Tue, 06 Feb 2018 14:46:07 +0000 + +citus (7.2.0.citus-1) stable; urgency=low + + * Adds support for CTEs + + * Adds support for subqueries that require merge step + + * Adds support for set operations (UNION, INTERSECT, ...) + + * Adds support for 2PC auto-recovery + + * Adds support for querying local tables in CTEs and subqueries + + * Adds support for more SQL coverage in subqueries for reference tables + + * Adds support for count(distinct) in queries with a subquery + + * Adds support for non-equijoins when there is already an equijoin + + * Adds support for real-time executor to run in transaction blocks + + * Adds infrastructure for storing intermediate distributed query results + + * Adds a new GUC named enable_repartition_joins for auto executor switch + + * Adds support for limiting the intermediate result size + + * Improves support for queries with unions containing filters + + * Improves support for queries with unions containing joins + + * Improves support for subqueries in the WHERE clause + + * Increases COPY throughput + + * Enables pushing down queries containing only recurring tuples and GROUP BY + + * Load-balance queries that read from 0 shards + + * Improves support for using functions in subqueries + + * Fixes a bug that causing real-time executor to crash during cancellation + + * Fixes a bug that causing real-time executor to get stuck on cancellation + + * Fixes a bug that could block modification queries unnecessarily + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Fixes a bug that could cause an assert failure with ANALYZE statements + + * Fixes a bug that would push down wrong set operations in subqueries + + * Fixes a bug that could cause a deadlock in create_distributed_table + + * Fixes a bug that could confuse user about ANALYZE usage + + * Fixes a bug causing false positive distributed deadlock detections + + * Relaxes the locking for DDL commands on partitioned tables + + * Relaxes the locking on COPY with replication + + * Logs more remote commands when citus.log_remote_commands is set + + -- Burak Velioglu Tue, 16 Jan 2018 14:34:20 +0000 + +citus (6.2.5.citus-1) stable; urgency=low + + * Fixes a bug that could crash the coordinator while reporting a remote error + + -- Burak Velioglu Thu, 11 Jan 2018 11:40:28 +0000 + +citus (7.1.2.citus-1) stable; urgency=low + + * Fixes a bug that could cause assigning wrong IDs to transactions + + * Increases COPY throughput + + -- Burak Velioglu Fri, 05 Jan 2018 09:00:07 +0000 + +citus (7.1.1.citus-1) stable; urgency=low + + * Fixes a bug preventing pushing down subqueries with reference tables + + * Fixes a bug that could create false positive distributed deadlocks + + * Fixes a bug that could prevent running concurrent COPY and multi-shard DDL + + * Fixes a bug that could mislead users about ANALYZE queries + + -- Burak Velioglu Tue, 05 Dec 2017 09:00:07 +0000 + +citus (7.1.0.citus-1) stable; urgency=low + + * Adds support for native queries with multi shard UPDATE/DELETE queries + + * Expands reference table support in subquery pushdown + + * Adds window function support for subqueries and INSERT ... SELECT queries + + * Adds support for COUNT(DISTINCT) [ON] queries on non-partition columns + + * Adds support for DISTINCT [ON] queries on non-partition columns + + * Introduces basic usage statistic collector + + * Adds support for setting replica identity while creating distributed tables + + * Adds support for ALTER TABLE ... REPLICA IDENTITY queries + + * Adds pushdown support for LIMIT and HAVING grouped by partition key + + * Adds support for INSERT ... SELECT queries via worker nodes on MX clusters + + * Adds support for adding primary key using already defined index + + * Adds replication parameter to shard copy functions + + * Changes shard_name UDF to omit public schema name + + * Adds master_move_node UDF to make changes on nodename/nodeport more easy + + * Fixes a bug that could cause casting error with INSERT ... SELECT queries + + * Fixes a bug that could prevent upgrading servers from Citus 6.1 + + * Fixes a bug that could prevent attaching partitions to a table in schema + + * Fixes a bug preventing adding nodes to clusters with reference tables + + * Fixes a bug that could cause a crash with INSERT ... SELECT queries + + * Fixes a bug that could prevent creating a partitoned table on Cloud + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Velioglu Wed, 15 Nov 2017 09:00:07 +0000 + +citus (7.0.3.citus-1) stable; urgency=low + + * Fixes several bugs that could cause crash + + * Fixes a bug that could cause deadlock while creating reference tables + + * Fixes a bug that could cause false-positives in deadlock detection + + * Fixes a bug that could cause 2PC recovery not to work from MX workers + + * Fixes a bug that could cause cache incohorency + + * Fixes a bug that could cause maintenance daemon to skip cache invalidations + + * Improves performance of transaction recovery by using correct index + + -- Burak Yucesoy Mon, 16 Oct 2017 11:52:07 +0000 + +citus (7.0.2.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 22:29:01 +0000 + +citus (6.2.4.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 21:31:35 +0000 + +citus (6.1.3.citus-1) stable; urgency=low + + * Updates task-tracker to limit file access + + -- Burak Yucesoy Thu, 28 Sep 2017 20:31:35 +0000 + +citus (7.0.1.citus-1) stable; urgency=low + + * Fixes a bug that could cause memory leaks in INSERT ... SELECT queries + + * Fixes a bug that could cause incorrect execution of prepared statements + + * Fixes a bug that could cause excessive memory usage during COPY + + * Incorporates latest changes from core PostgreSQL code + + -- Burak Yucesoy Tue, 12 Sep 2017 17:53:50 +0000 + +citus (7.0.0.citus-1) stable; urgency=low + + * Adds support for PostgreSQL 10 + + * Drops support for PostgreSQL 9.5 + + * Adds support for multi-row INSERT + + * Adds support for router UPDATE and DELETE queries with subqueries + + * Adds infrastructure for distributed deadlock detection + + * Deprecates enable_deadlock_prevention flag + + * Adds support for partitioned tables + + * Adds support for creating UNLOGGED tables + + * Adds support for SAVEPOINT + + * Adds UDF citus_create_restore_point for taking distributed snapshots + + * Adds support for evaluating non-pushable INSERT ... SELECT queries + + * Adds support for subquery pushdown on reference tables + + * Adds shard pruning support for IN and ANY + + * Adds support for UPDATE and DELETE commands that prune down to 0 shard + + * Enhances transaction support by relaxing some transaction restrictions + + * Fixes a bug causing crash if distributed table has no shards + + * Fixes a bug causing crash when removing inactive node + + * Fixes a bug causing failure during COPY on tables with dropped columns + + * Fixes a bug causing failure during DROP EXTENSION + + * Fixes a bug preventing executing VACUUM and INSERT concurrently + + * Fixes a bug in prepared INSERT statements containing an implicit cast + + * Fixes several issues related to statement cancellations and connections + + * Fixes several 2PC related issues + + * Removes an unnecessary dependency causing warning messages in pg_dump + + * Adds internal infrastructure for follower clusters + + * Adds internal infrastructure for progress tracking + + * Implements various performance improvements + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Mon, 28 Aug 2017 12:27:53 +0000 + +citus (6.2.3.citus-1) stable; urgency=low + + * Fixes a crash during execution of local CREATE INDEX CONCURRENTLY + + * Fixes a bug preventing usage of quoted column names in COPY + + * Fixes a bug in prepared INSERTs with implicit cast in partition column + + * Relaxes locks in VACUUM to ensure concurrent execution with INSERT + + -- Burak Yucesoy Thu, 13 Jul 2017 11:27:17 +0000 + +citus (6.2.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Burak Velioglu Wed, 07 Jun 2017 09:42:17 +0000 + +citus (6.1.2.citus-1) stable; urgency=low + + * Fixes a common cause of deadlocks when repairing tables with foreign keys + + -- Jason Petersen Wed, 31 May 2017 16:14:11 +0000 + +citus (6.2.1.citus-1) stable; urgency=low + + * Relaxes version-check logic to avoid breaking non-distributed commands + + -- Jason Petersen Wed, 24 May 2017 22:36:07 +0000 + +citus (6.2.0.citus-1) stable; urgency=low + + * Increases SQL subquery coverage by pushing down more kinds of queries + + * Adds CustomScan API support to allow read-only transactions + + * Adds support for CREATE/DROP INDEX CONCURRENTLY + + * Adds support for ALTER TABLE ... ADD CONSTRAINT + + * Adds support for ALTER TABLE ... RENAME COLUMN + + * Adds support for DISABLE/ENABLE TRIGGER ALL + + * Adds support for expressions in the partition column in INSERTs + + * Adds support for query parameters in combination with function evaluation + + * Adds support for creating distributed tables from non-empty local tables + + * Adds UDFs to get size of distributed tables + + * Adds UDFs to add a new node without replicating reference tables + + * Adds checks to prevent running Citus binaries with wrong metadata tables + + * Improves shard pruning performance for range queries + + * Improves planner performance for joins involving co-located tables + + * Improves shard copy performance by creating indexes after copy + + * Improves task-tracker performance by batching several status checks + + * Enables router planner for queries on range partitioned table + + * Changes TRUNCATE to drop local data only if enable_ddl_propagation is off + + * Starts to execute DDL on coordinator before workers + + * Fixes a bug causing incorrectly reading invalidated cache + + * Fixes a bug related to creation of schemas in workers with incorrect owner + + * Fixes a bug related to concurrent run of shard drop functions + + * Fixes a bug related to EXPLAIN ANALYZE with DML queries + + * Fixes a bug related to SQL functions in FROM clause + + * Adds a GUC variable to report cross shard queries + + * Fixes a bug related to partition columns without native hash function + + * Adds internal infrastructures and tests to improve development process + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Tue, 16 May 2017 16:05:22 +0000 + +citus (6.1.1.citus-1) stable; urgency=low + + * Fixes a crash caused by router executor use after connection timeouts + + * Fixes a crash caused by relation cache invalidation during COPY + + * Fixes bug related to DDL use within PL/pgSQL functions + + * Fixes a COPY bug related to types lacking binary output functions + + * Fixes a bug related to modifications with parameterized partition values + + * Fixes improper value interpolation in worker sequence generation + + * Guards shard pruning logic against zero-shard tables + + * Fixes possible NULL pointer dereference and buffer underflow, via PVS-Studio + + * Fixes a INSERT...SELECT bug that could push down non-partition column JOINs + + -- Metin Doslu Fri, 5 May 2017 17:42:00 +0000 + +citus (6.1.0.citus-1) stable; urgency=low + + * Implements reference tables, transactionally replicated to all nodes + + * Adds upgrade_to_reference_table UDF to upgrade pre-6.1 reference tables + + * Expands prepared statement support to nearly all statements + + * Adds support for creating VIEWs which reference distributed tables + + * Adds targeted VACUUM/ANALYZE support + + * Adds support for the FILTER clause in aggregate expressions + + * Adds support for function evaluation within INSERT INTO ... SELECT + + * Adds support for creating foreign key constraints with ALTER TABLE + + * Adds logic to choose router planner for all queries it supports + + * Enhances create_distributed_table with parameter for explicit colocation + + * Adds generally useful utility UDFs previously available as "Citus Tools" + + * Adds user-facing UDFs for locking shard resources and metadata + + * Refactors connection and transaction management for more consistency + + * Enhances COPY with fully transactional semantics + + * Improves support for cancellation for a number of queries and commands + + * Adds column_to_column_name UDF to help users understand partkey values + + * Adds master_disable_node UDF for temporarily disabling nodes + + * Adds proper MX ("masterless") metadata propagation logic + + * Adds start_metadata_sync_to_node UDF to propagate metadata changes to nodes + + * Enhances SERIAL compatibility with MX tables + + * Adds an node_connection_timeout parameter to set node connection timeouts + + * Adds enable_deadlock_prevention setting to permit multi-node transactions + + * Adds a replication_model setting to specify replication of new tables + + * Changes the shard_replication_factor setting's default value to one + + * Adds code to automatically set max_prepared_transactions if not configured + + * Accelerates lookup of colocated shard placements + + * Fixes a bug affecting INSERT INTO ... SELECT queries using constant values + + * Fixes a bug by ensuring COPY does not mark placements inactive + + * Fixes a bug affecting reads from pg_dist_shard_placement table + + * Fixes a crash triggered by creating a foreign key without a column + + * Fixes a crash related to accessing catalog tables after aborted transaction + + * Fixes a bug affecting JOIN queries requiring repartitions + + * Fixes a bug affecting node insertions to pg_dist_node table + + * Fixes a crash triggered by queries with modifying common table expressions + + * Fixes a bug affecting workloads with concurrent shard appends and deletions + + * Addresses various race conditions and deadlocks + + * Improves and standardizes error messages + + -- Burak Yucesoy Thu, 9 Feb 2017 16:17:41 +0000 + +citus (6.0.1.citus-3) stable; urgency=low + + * First build using new versioning practices + + -- Jason Petersen Wed, 8 Feb 2017 23:19:46 +0000 + +citus (6.0.1.citus-2) stable; urgency=low + + * Transitional package to guide users to new package name + + -- Jason Petersen Mon, 6 Feb 2017 16:33:44 +0000 + +citus (6.0.1.citus-1) stable; urgency=low + + * Fixes a bug causing failures during pg_upgrade + + * Fixes a bug preventing DML queries during colocated table creation + + * Fixes a bug that caused NULL parameters to be incorrectly passed as text + + -- Burak Yucesoy Wed, 30 Nov 2016 15:27:38 +0000 + +citus (6.0.0.citus-1) stable; urgency=low + + * Adds compatibility with PostgreSQL 9.6, now the recommended version + + * Removes the pg_worker_list.conf file in favor of a pg_dist_node table + + * Adds master_add_node and master_add_node UDFs to manage membership + + * Removes the \stage command and corresponding csql binary in favor of COPY + + * Removes copy_to_distributed_table in favor of first-class COPY support + + * Adds support for multiple DDL statements within a transaction + + * Adds support for certain foreign key constraints + + * Adds support for parallel INSERT INTO ... SELECT against colocated tables + + * Adds support for the TRUNCATE command + + * Adds support for HAVING clauses in SELECT queries + + * Adds support for EXCLUDE constraints which include the partition column + + * Adds support for system columns in queries (tableoid, ctid, etc.) + + * Adds support for relation name extension within INDEX definitions + + * Adds support for no-op UPDATEs of the partition column + + * Adds several general-purpose utility UDFs to aid in Citus maintenance + + * Adds master_expire_table_cache UDF to forcibly expire cached shards + + * Parallelizes the processing of DDL commands which affect distributed tables + + * Adds support for repartition jobs using composite or custom types + + * Enhances object name extension to handle long names and large shard counts + + * Parallelizes the master_modify_multiple_shards UDF + + * Changes distributed table creation to error if target table is not empty + + * Changes the pg_dist_shard.logicalrelid column from an oid to regclass + + * Adds a placementid column to pg_dist_shard_placement, replacing Oid use + + * Removes the pg_dist_shard.shardalias distribution metadata column + + * Adds pg_dist_partition.repmodel to track tables using streaming replication + + * Adds internal infrastructure to take snapshots of distribution metadata + + * Addresses the need to invalidate prepared statements on metadata changes + + * Adds a mark_tables_colocated UDF for denoting pre-6.0 manual colocation + + * Fixes a bug affecting prepared statement execution within PL/pgSQL + + * Fixes a bug affecting COPY commands using composite types + + * Fixes a bug that could cause crashes during EXPLAIN EXECUTE + + * Separates worker and master job temporary folders + + * Eliminates race condition between distributed modification and repair + + * Relaxes the requirement that shard repairs also repair colocated shards + + * Implements internal functions to track which tables' shards are colocated + + * Adds pg_dist_partition.colocationid to track colocation group membership + + * Extends shard copy and move operations to respect colocation settings + + * Adds pg_dist_local_group to prepare for future MX-related changes + + * Adds create_distributed_table to easily create shards and infer colocation + + -- Jason Petersen Tue, 8 Nov 2016 19:45:45 +0000 + +citus (5.2.2.citus-1) stable; urgency=low + + * Adds support for IF NOT EXISTS clause of CREATE INDEX command + + * Adds support for RETURN QUERY and FOR ... IN PL/pgSQL features + + * Extends the router planner to handle more queries + + * Changes COUNT of zero-row sets to return 0 rather than an empty result + + * Reduces the minimum permitted task_tracker_delay to a single millisecond + + * Fixes a bug that caused crashes during joins with a WHERE false clause + + * Fixes a bug triggered by unique violation errors raised in long txns + + * Fixes a bug resulting in multiple registration of transaction callbacks + + * Fixes a bug which could result in stale reads of distribution metadata + + * Fixes a bug preventing distributed modifications in some PL/pgSQL functions + + * Fixes some code paths that could hypothetically read uninitialized memory + + * Lowers log level of "waiting for activity" messages + + -- Jason Petersen Tue, 8 Nov 2016 18:43:37 +0000 + +citus (5.2.1.citus-1) stable; urgency=low + + * Fixes subquery pushdown to properly extract outer join qualifiers + + * Addresses possible memory leak during multi-shard transactions + + -- Jason Petersen Tue, 6 Sep 2016 20:47:15 +0000 + +citus (5.2.0.citus-1) stable; urgency=low + + * Drops support for PostgreSQL 9.4; PostgreSQL 9.5 is required + + * Adds schema support for tables, named objects (types, operators, etc.) + + * Evaluates non-immutable functions on master in all modification commands + + * Adds support for SERIAL types in non-partition columns + + * Adds support for RETURNING clause in INSERT, UPDATE, and DELETE commands + + * Adds support for multi-statement transactions using a fixed set of nodes + + * Full SQL support for SELECT queries which can be executed on single worker + + * Adds option to perform DDL changes using prepared transactions (2PC) + + * Adds an enable_ddl_propagation parameter to control DDL propagation + + * Accelerates shard pruning during merges + + * Adds master_modify_multiple_shards UDF to modify many shards at once + + * Adds COPY support for arrays of user-defined types + + * Now supports parameterized prepared statements for certain use cases + + * Extends LIMIT/OFFSET support to all executor types + + * Constraint violations now fail fast rather than hitting all placements + + * Makes master_create_empty_shard aware of shard placement policy + + * Reduces unnecessary sleep during queries processed by real-time executor + + * Improves task tracker executor's task cleanup logic + + * Relaxes restrictions on cancellation of DDL commands + + * Removes ONLY keyword from worker SELECT queries + + * Error message improvements and standardization + + * Moves master_update_shard_statistics function to pg_catalog schema + + * Fixes a bug where hash-partitioned anti-joins could return bad results + + * Now sets storage type correctly for foreign table-backed shards + + * Fixes master_update_shard_statistics issue with hash-partitioned tables + + * Fixes an issue related to extending table names that require escaping + + * Reduces risk of row counter overflows during modifications + + * Fixes a crash related to FILTER clause use in COUNT DISTINCT subqueries + + * Fixes crashes related to partition columns with high attribute numbers + + * Fixes certain subquery and join crashes + + * Detects flex for build even if PostgreSQL was built without it + + * Fixes assert-enabled crash when all_modifications_commutative is true + + -- Jason Petersen Wed, 17 Aug 2016 10:23:21 +0000 + +citus (5.2.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.2. + + -- Jason Petersen Mon, 01 Aug 2016 17:01:05 +0000 + +citus (5.1.1-1) stable; urgency=low + + * Adds complex count distinct expression support in repartitioned subqueries + + * Improves task tracker job cleanup logic, addressing a memory leak + + * Fixes bug that generated incorrect results for LEFT JOIN queries + + * Improves compatibility with Debian's reproducible builds project + + * Fixes build issues on FreeBSD platforms + + -- Jason Petersen Fri, 17 Jun 2016 16:20:15 +0000 + +citus (5.1.0-1) stable; urgency=low + + * Adds distributed COPY to rapidly populate distributed tables + + * Adds support for using EXPLAIN on distributed queries + + * Recognizes and fast-paths single-shard SELECT statements automatically + + * Increases INSERT throughput via shard pruning optimizations + + * Improves planner performance for joins involving tables with many shards + + * Adds ability to pass columns as arguments to function calls in UPDATEs + + * Introduces transaction manager for use by multi-shard commands + + * Adds COUNT(DISTINCT ...) pushdown optimization for hash-partitioned tables + + * Adds support for some UNIQUE indexes on hash- or range-partitioned tables + + * Deprecates \stage in favor of using COPY for append-partition tables + + * Deprecates copy_to_distributed_table in favor of first-class COPY support + + * Fixes build problems when using non-packaged PostgreSQL installs + + * Fixes bug that sometimes skipped pruning when partitioned by VARCHAR column + + * Fixes bug impeding use of user functions in repartitioned subqueries + + * Fixes bug involving queries with equality comparisons of boolean types + + * Fixes crash that prevented use alongside pg_stat_statements + + * Fixes crash arising from SELECT queries that lack a target list + + * Improves warning and error messages + + -- Jason Petersen Tue, 17 May 2016 16:55:02 +0000 + +citus (5.1.0~rc.2-1) testing; urgency=low + + * Fix EXPLAIN output when FORMAT JSON in use + + -- Jason Petersen Mon, 16 May 2016 11:08:09 +0000 + +citus (5.1.0~rc.1-1) testing; urgency=low + + * Release candidate for 5.1. + + -- Jason Petersen Wed, 04 May 2016 19:26:23 +0000 + +citus (5.0.1-1) stable; urgency=low + + * Fixes issues on 32-bit systems + + -- Jason Petersen Fri, 15 Apr 2016 19:17:35 +0000 + +citus (5.0.0-1) stable; urgency=low + + * Initial release + + -- Jason Petersen Thu, 24 Mar 2016 10:12:52 -0400 diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index d8416bd7..8541ed93 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -3,7 +3,7 @@ from datetime import datetime from shutil import copyfile -from .test_utils import * +from .test_utils import are_strings_equal from ..update_package_properties import * TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -57,14 +57,28 @@ def test_prepend_latest_changelog_into_debian_changelog(self): refer_file_path = f"{TEST_BASE_PATH}/files/debian.changelog.refer" changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" copyfile(refer_file_path, changelog_file_path) - changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) try: - prepend_latest_changelog_into_debian_changelog(changelog, PROJECT_VERSION, True, 1, changelog_file_path, + prepend_latest_changelog_into_debian_changelog(latest_changelog, PROJECT_VERSION, True, 1, + changelog_file_path, MICROSOFT_EMAIL, NAME_SURNAME, CHANGELOG_DATE) self.verify_prepend_debian_changelog(changelog_file_path) finally: os.remove(changelog_file_path) + def test_prepend_latest_changelog_into_debian_changelog_10_0_3_already_included(self): + refer_file_path = f"{TEST_BASE_PATH}/files/debian.changelog_include_10_0_3.refer" + changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" + copyfile(refer_file_path, changelog_file_path) + latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + try: + prepend_latest_changelog_into_debian_changelog(latest_changelog, PROJECT_VERSION, True, 1, + changelog_file_path, + MICROSOFT_EMAIL, NAME_SURNAME, CHANGELOG_DATE) + self.assertRaises(ValueError) + finally: + os.remove(changelog_file_path) + def verify_prepend_debian_changelog(self, changelog_file_path): with open(changelog_file_path, "r") as reader: content = reader.read() @@ -96,6 +110,20 @@ def test_update_rpm_spec(self): copyfile(spec_file_copy, spec_file) os.remove(spec_file_copy) + def test_update_rpm_spec_include_10_0_3(self): + project_name = "citus" + spec_file = f"{TEST_BASE_PATH}/files/citus_include_10_0_3.spec" + spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" + templates_path = f"{BASE_PATH}/templates" + copyfile(spec_file, spec_file_copy) + try: + update_rpm_spec(project_name, PROJECT_VERSION, MICROSOFT_EMAIL, NAME_SURNAME, True, 1, + spec_file, CHANGELOG_DATE, templates_path) + self.assertRaises(ValueError) + finally: + copyfile(spec_file_copy, spec_file) + os.remove(spec_file_copy) + def verify_rpm_spec(self, spec_file_reference, spec_file_for_test): with open(spec_file_for_test, "r") as reader_test: with open(spec_file_reference, "r") as reader_reference: diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 06636f91..24d4f4fe 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -88,25 +88,15 @@ def find_nth_overlapping_line_by_regex(subject_string, regex_pattern, n) -> int: return index -# def find_nth_overlapping_regex(subject_string, search_string, n): -# re. -# start = subject_string.find(search_string) -# -# while start >= 0 and n > 1: -# start = subject_string.find(search_string, start + 1) -# n -= 1 -# return start - - def get_last_changelog_content(all_changelog_content: str) -> str: second_changelog_index = find_nth_overlapping(all_changelog_content, "###", 3) changelogs = all_changelog_content[:second_changelog_index] lines = changelogs.splitlines() if len(lines) < 1: - raise Exception("At least one line should be in changelog") + raise ValueError("At least one line should be in changelog") changelog_header = lines[0] if not changelog_header.startswith("###"): - raise Exception("Changelog header should start with '###'") + raise ValueError("Changelog header should start with '###'") return changelogs @@ -115,7 +105,7 @@ def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: lines = all_changelog_content.splitlines() changelogs = "".join([str(elem) + "\n" for elem in lines[:second_changelog_index - 1]]) if len(lines) < 1: - raise Exception("At least one line should be in changelog") + raise ValueError("At least one line should be in changelog") return changelogs @@ -141,7 +131,7 @@ def get_debian_changelog_header(changelog_header: is_project_changelog_header(st parentheses_removed_string = remove_parentheses_from_string(hash_removed_string) words = parentheses_removed_string.strip().split(" ") if len(words) != 2: - raise Exception("Two words should be included in striped version header") + raise ValueError("Two words should be included in striped version header") project_name = words[0] project_version = words[1].lstrip("v") version_on_changelog = get_version_number_with_project_name(project_name, project_version, fancy, @@ -163,7 +153,7 @@ def convert_citus_changelog_into_debian_changelog(citus_changelog: str, fancy: b return debian_latest_changelog -def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, tag_name: str, fancy: bool, +def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, project_version: str, fancy: bool, fancy_version_number: int, changelog_file_path: str, microsoft_email: str, name_surname: str, changelog_date: date) -> None: @@ -171,7 +161,7 @@ def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, tag_na fancy_version_number, microsoft_email, name_surname, changelog_date) with open(changelog_file_path, "r+") as reader: - if not (f"({tag_name}" in reader.readline()): + if not (f"({project_version}" in reader.readline()): reader.seek(0, 0) old_changelog = reader.read() changelog = f"{debian_latest_changelog}{old_changelog}" @@ -181,7 +171,6 @@ def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, tag_na print("Already version in the debian changelog") - @validate_parameters def update_pkgvars(version: is_version(non_empty(no_whitespaces(non_blank(str)))), fancy: bool, fancy_release_count: non_negative(int), templates_path: str, pkgvars_path: str) -> None: @@ -246,12 +235,6 @@ def convert_citus_changelog_into_rpm_changelog(project_name: str, project_versio return rpm_changelog -# # print(get_rpm_changelog_history("citus")) -# changelog = get_changelog_for_tag("93647419346e194ae3094598139d68ebf2263ee0", "citus", "v10.0.3") -# print(get_rpm_changelog("citus", "10.0.3", changelog, "gindibay@microsoft.com", "Gürkan İndibay", True, 1)) -# print(get_rpm_header("citus", "gindibay@microsoft.com", "Gurkan Indibay", "10.0.3", True, 2)) - - def update_rpm_spec(project_name: str, project_version: str, microsoft_email: str, name_surname: str, fancy: bool, fancy_version_number: int, spec_file_name: str, changelog_date: date, templates_path: str) -> None: @@ -261,6 +244,11 @@ def update_rpm_spec(project_name: str, project_version: str, microsoft_email: st template = env.get_template('project.spec.tmpl') rpm_changelog_history = get_rpm_changelog_history(project_name, spec_file_name) + history_lines = rpm_changelog_history.splitlines() + + if len(history_lines) > 0 and project_version in history_lines[1]: + raise ValueError(f"{project_version} already exists in rpm spec file") + latest_changelog = convert_citus_changelog_into_rpm_changelog(project_name, project_version, microsoft_email, name_surname, fancy, fancy_version_number, changelog_date) @@ -272,14 +260,15 @@ def update_rpm_spec(project_name: str, project_version: str, microsoft_email: st @validate_parameters -def update_all_changes(github_token: non_empty(non_blank(str)), project_name: non_empty(str), project_version: is_version(str), +def update_all_changes(github_token: non_empty(non_blank(str)), project_name: non_empty(str), + project_version: is_version(str), tag_name: is_tag(str), fancy: bool, fancy_version_number: non_negative(int), microsoft_email: is_email(str), name_surname: non_empty(non_blank(str)), release_date: date, packaging_path: str): templates_path = f"{BASE_PATH}/templates" update_pkgvars(project_version, fancy, fancy_version_number, templates_path, f"{packaging_path}") latest_changelog = get_changelog_for_tag(github_token, project_name, tag_name) - prepend_latest_changelog_into_debian_changelog(latest_changelog, tag_name, fancy, fancy_version_number, + prepend_latest_changelog_into_debian_changelog(latest_changelog, project_version, fancy, fancy_version_number, f"{packaging_path}/debian/changelog", microsoft_email, name_surname, release_date) spec_file_name = f"{packaging_path}/{get_spec_file_name(project_name)}" From 9cf2e12c9ca4a601d9ea1b1e6a092534f81ba39c Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 2 Apr 2021 22:46:05 +0300 Subject: [PATCH 027/106] Remove unnecessary files --- python/templates/showpersons.txt | 3 - python/tests/to_be_deleted/actual.txt | 315 ------------------------- python/tests/to_be_deleted/expected | 318 -------------------------- 3 files changed, 636 deletions(-) delete mode 100644 python/templates/showpersons.txt delete mode 100644 python/tests/to_be_deleted/actual.txt delete mode 100644 python/tests/to_be_deleted/expected diff --git a/python/templates/showpersons.txt b/python/templates/showpersons.txt deleted file mode 100644 index dc32f91c..00000000 --- a/python/templates/showpersons.txt +++ /dev/null @@ -1,3 +0,0 @@ -{% for person in persons -%} - {{ person.name }} {{ person.age }} -{% endfor %} \ No newline at end of file diff --git a/python/tests/to_be_deleted/actual.txt b/python/tests/to_be_deleted/actual.txt deleted file mode 100644 index a783f596..00000000 --- a/python/tests/to_be_deleted/actual.txt +++ /dev/null @@ -1,315 +0,0 @@ -%global pgmajorversion 11 -%global pgpackageversion 11 -%global pginstdir /usr/pgsql-%{pgpackageversion} -%global sname citus - -Summary: PostgreSQL-based distributed RDBMS -Name: %{sname}%{?pkginfix}_%{pgmajorversion} -Provides: %{sname}_%{pgmajorversion} -Conflicts: %{sname}_%{pgmajorversion} -Version: 10.0.3-1 -Release: 1%{dist} -License: AGPLv3 -Group: Applications/Databases -Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz -URL: https://github.com/citusdata/ -BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel -Requires: postgresql%{pgmajorversion}-server -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) - -%description -Citus horizontally scales PostgreSQL across commodity servers -using sharding and replication. Its query engine parallelizes -incoming SQL queries across these servers to enable real-time -responses on large datasets. - -Citus extends the underlying database rather than forking it, -which gives developers and enterprises the power and familiarity -of a traditional relational database. As an extension, Citus -supports new PostgreSQL releases, allowing users to benefit from -new features while maintaining compatibility with existing -PostgreSQL tools. Note that Citus supports many (but not all) SQL -commands. - -%prep -%setup -q -n %{sname}-%{version} - -%build -%configure PG_CONFIG=%{pginstdir}/bin/pg_config --with-extra-version="%{?conf_extra_version}" --with-security-flags -make %{?_smp_mflags} - -%install -%make_install -# Install documentation with a better name: -%{__mkdir} -p %{buildroot}%{pginstdir}/doc/extension -%{__cp} README.md %{buildroot}%{pginstdir}/doc/extension/README-%{sname}.md -# Set paths to be packaged other than LICENSE, README & CHANGELOG.md -echo %{pginstdir}/include/server/citus_*.h >> installation_files.list -echo %{pginstdir}/include/server/distributed/*.h >> installation_files.list -echo %{pginstdir}/lib/%{sname}.so >> installation_files.list -echo %{pginstdir}/share/extension/%{sname}-*.sql >> installation_files.list -echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list -%ifarch ppc64 ppc64le - %else - %if 0%{?rhel} && 0%{?rhel} <= 6 - %else - echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list - echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list - echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list - - # Columnar does not exist in Citus versions < 10.0 - # At this point, we don't have %{pginstdir}, - # so first check build directory for columnar. - [[ -d %{buildroot}%{pginstdir}/lib/bitcode/columnar/ ]] && echo %{pginstdir}/lib/bitcode/columnar/*.bc >> installation_files.list - %endif -%endif - -%clean -%{__rm} -rf %{buildroot} - -%files -f installation_files.list -%files -%defattr(-,root,root,-) -%doc CHANGELOG.md -%if 0%{?rhel} && 0%{?rhel} <= 6 -%doc LICENSE -%else -%license LICENSE -%endif -%doc %{pginstdir}/doc/extension/README-%{sname}.md - -%changelog -* Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 -- Official 10.0.2 release of Citus - -* Wed Jan 27 2021 - gurkanindibay 9.5.2.citus-1 -- Official 9.5.2 release of Citus - -* Tue Jan 5 2021 - gurkanindibay 9.4.4.citus-1 -- Official 9.4.4 release of Citus - -* Wed Dec 2 2020 - Onur Tirtir 9.5.1.citus-1 -- Official 9.5.1 release of Citus - -* Tue Nov 24 2020 - Onur Tirtir 9.4.3.citus-1 -- Official 9.4.3 release of Citus - -* Wed Nov 11 2020 - Onur Tirtir 9.5.0.citus-1 -- Official 9.5.0 release of Citus - -* Thu Oct 22 2020 - Onur Tirtir 9.4.2.citus-1 -- Official 9.4.2 release of Citus - -* Wed Sep 30 2020 - Onur Tirtir 9.4.1.citus-1 -- Official 9.4.1 release of Citus - -* Tue Jul 28 2020 - Onur Tirtir 9.4.0.citus-1 -- Official 9.4.0 release of Citus - -* Mon Jul 27 2020 - Onur Tirtir 9.3.5.citus-1 -- Official 9.3.5 release of Citus - -* Wed Jul 22 2020 - Onur Tirtir 9.3.4.citus-1 -- Official 9.3.4 release of Citus - -* Mon Jul 13 2020 - Onur Tirtir 9.3.3.citus-1 -- Official 9.3.3 release of Citus - -* Thu May 7 2020 - Onur Tirtir 9.3.0.citus-1 -- Update to Citus 9.3.0 - -* Tue Mar 31 2020 - Onur Tirtir 9.2.4.citus-1 -- Update to Citus 9.2.4 - -* Thu Mar 26 2020 - Onur Tirtir 9.2.3.citus-1 -- Update to Citus 9.2.3 - -* Fri Mar 6 2020 - Onur Tirtir 9.0.2.citus-1 -- Update to Citus 9.0.2 - -* Fri Mar 6 2020 - Onur Tirtir 9.2.2.citus-1 -- Update to Citus 9.2.2 - -* Fri Feb 14 2020 - Onur Tirtir 9.2.1.citus-1 -- Update to Citus 9.2.1 - -* Mon Feb 10 2020 - Onur Tirtir 9.2.0.citus-1 -- Update to Citus 9.2.0 - -* Wed Dec 18 2019 - Onur Tirtir 9.1.1.citus-1 -- Update to Citus 9.1.1 - -* Thu Nov 28 2019 - Onur Tirtir 9.1.0.citus-1 -- Update to Citus 9.1.0 - -* Wed Oct 30 2019 - Hanefi Onaldi 9.0.1.citus-1 -- Update to Citus 9.0.1 - -* Thu Oct 10 2019 - Hanefi Onaldi 9.0.0.citus-1 -- Update to Citus 9.0.0 - -* Fri Aug 9 2019 - Hanefi Onaldi 8.3.2.citus-1 -- Update to Citus 8.3.2 - -* Mon Jul 29 2019 - Hanefi Onaldi 8.3.1.citus-1 -- Update to Citus 8.3.1 - -* Wed Jul 10 2019 - Burak Velioglu 8.3.0.citus-1 -- Update to Citus 8.3.0 - -* Wed Jun 12 2019 - Burak Velioglu 8.2.2.citus-1 -- Update to Citus 8.2.2 - -* Wed Apr 3 2019 - Burak Velioglu 8.2.1.citus-1 -- Update to Citus 8.2.1 - -* Wed Apr 3 2019 - Burak Velioglu 8.1.2.citus-1 -- Update to Citus 8.1.2 - -* Thu Mar 28 2019 - Burak Velioglu 8.2.0.citus-1 -- Update to Citus 8.2.0 - -* Wed Jan 9 2019 - Burak Velioglu 8.0.3.citus-1 -- Update to Citus 8.0.3 - -* Mon Jan 7 2019 - Burak Velioglu 8.1.1.citus-1 -- Update to Citus 8.1.1 - -* Tue Dec 18 2018 - Burak Velioglu 8.1.0.citus-1 -- Update to Citus 8.1.0 - -* Thu Dec 13 2018 - Burak Velioglu 8.0.2.citus-1 -- Update to Citus 8.0.2 - -* Wed Dec 12 2018 - Burak Velioglu 7.5.4.citus-1 -- Update to Citus 7.5.4 - -* Wed Nov 28 2018 - Burak Velioglu 8.0.1.citus-1 -- Update to Citus 8.0.1 - -* Wed Nov 28 2018 - Burak Velioglu 7.5.3.citus-1 -- Update to Citus 7.5.3 - -* Wed Nov 14 2018 - Burak Velioglu 7.5.2.citus-1 -- Update to Citus 7.5.2 - -* Fri Nov 02 2018 - Burak Velioglu 8.0.0.citus-1 -- Update to Citus 8.0.0 - -* Wed Aug 29 2018 - Burak Velioglu 7.5.1.citus-1 -- Update to Citus 7.5.1 - -* Fri Jul 27 2018 - Mehmet Furkan Sahin 7.4.2.citus-1 -- Update to Citus 7.4.2 - -* Wed Jul 25 2018 - Mehmet Furkan Sahin 7.5.0.citus-1 -- Update to Citus 7.5.0 - -* Wed Jun 20 2018 - Burak Velioglu 7.4.1.citus-1 -- Update to Citus 7.4.1 - -* Thu May 17 2018 - Burak Velioglu 7.2.2.citus-1 -- Update to Citus 7.2.2 - -* Tue May 15 2018 - Burak Velioglu 7.4.0.citus-1 -- Update to Citus 7.4.0 - -* Thu Mar 15 2018 - Burak Velioglu 7.3.0.citus-1 -- Update to Citus 7.3.0 - -* Tue Feb 6 2018 - Burak Velioglu 7.2.1.citus-1 -- Update to Citus 7.2.1 - -* Tue Jan 16 2018 - Burak Velioglu 7.2.0.citus-1 -- Update to Citus 7.2.0 - -* Thu Jan 11 2018 - Burak Velioglu 6.2.5.citus-1 -- Update to Citus 6.2.5 - -* Fri Jan 05 2018 - Burak Velioglu 7.1.2.citus-1 -- Update to Citus 7.1.2 - -* Tue Dec 05 2017 - Burak Velioglu 7.1.1.citus-1 -- Update to Citus 7.1.1 - -* Wed Nov 15 2017 - Burak Velioglu 7.1.0.citus-1 -- Update to Citus 7.1.0 - -* Mon Oct 16 2017 - Burak Yucesoy 7.0.3.citus-1 -- Update to Citus 7.0.3 - -* Thu Sep 28 2017 - Burak Yucesoy 7.0.2.citus-1 -- Update to Citus 7.0.2 - -* Thu Sep 28 2017 - Burak Yucesoy 6.2.4.citus-1 -- Update to Citus 6.2.4 - -* Thu Sep 28 2017 - Burak Yucesoy 6.1.3.citus-1 -- Update to Citus 6.1.3 - -* Tue Sep 12 2017 - Burak Yucesoy 7.0.1.citus-1 -- Update to Citus 7.0.1 - -* Tue Aug 29 2017 - Burak Yucesoy 7.0.0.citus-1 -- Update to Citus 7.0.0 - -* Thu Jul 13 2017 - Burak Yucesoy 6.2.3.citus-1 -- Update to Citus 6.2.3 - -* Wed Jun 7 2017 - Burak Velioglu 6.2.2.citus-1 -- Update to Citus 6.2.2 - -* Wed Jun 7 2017 - Jason Petersen 6.1.2.citus-1 -- Update to Citus 6.1.2 - -* Wed May 24 2017 - Jason Petersen 6.2.1.citus-1 -- Update to Citus 6.2.1 - -* Tue May 16 2017 - Burak Yucesoy 6.2.0.citus-1 -- Update to Citus 6.2.0 - -* Fri May 5 2017 - Metin Doslu 6.1.1.citus-1 -- Update to Citus 6.1.1 - -* Thu Feb 9 2017 - Burak Yucesoy 6.1.0.citus-1 -- Update to Citus 6.1.0 - -* Wed Feb 8 2017 - Jason Petersen 6.0.1.citus-2 -- Transitional package to guide users to new package name - -* Wed Nov 30 2016 - Burak Yucesoy 6.0.1.citus-1 -- Update to Citus 6.0.1 - -* Tue Nov 8 2016 - Jason Petersen 6.0.0.citus-1 -- Update to Citus 6.0.0 - -* Tue Nov 8 2016 - Jason Petersen 5.2.2.citus-1 -- Update to Citus 5.2.2 - -* Tue Sep 6 2016 - Jason Petersen 5.2.1.citus-1 -- Update to Citus 5.2.1 - -* Wed Aug 17 2016 - Jason Petersen 5.2.0.citus-1 -- Update to Citus 5.2.0 - -* Mon Aug 1 2016 - Jason Petersen 5.2.0-0.1.rc.1 -- Release candidate for 5.2 - -* Fri Jun 17 2016 - Jason Petersen 5.1.1-1 -- Update to Citus 5.1.1 - -* Tue May 17 2016 - Jason Petersen 5.1.0-1 -- Update to Citus 5.1.0 - -* Mon May 16 2016 - Jason Petersen 5.1.0-0.2.rc.2 -- Fix EXPLAIN output when FORMAT JSON in use - -* Wed May 4 2016 - Jason Petersen 5.1.0-0.1.rc.1 -- Release candidate for 5.1 - -* Fri Apr 15 2016 - Jason Petersen 5.0.1-1 -- Fixes issues on 32-bit systems - -* Fri Mar 25 2016 - Devrim Gündüz 5.0.0-1 -- Initial RPM packaging for PostgreSQL RPM Repository, - based on the spec file of Jason Petersen @ Citus. diff --git a/python/tests/to_be_deleted/expected b/python/tests/to_be_deleted/expected deleted file mode 100644 index 1d98bfa5..00000000 --- a/python/tests/to_be_deleted/expected +++ /dev/null @@ -1,318 +0,0 @@ -%global pgmajorversion 11 -%global pgpackageversion 11 -%global pginstdir /usr/pgsql-%{pgpackageversion} -%global sname citus - -Summary: PostgreSQL-based distributed RDBMS -Name: %{sname}%{?pkginfix}_%{pgmajorversion} -Provides: %{sname}_%{pgmajorversion} -Conflicts: %{sname}_%{pgmajorversion} -Version: 10.0.3-1 -Release: 1%{dist} -License: AGPLv3 -Group: Applications/Databases -Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz -URL: https://github.com/citusdata/ -BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel -Requires: postgresql%{pgmajorversion}-server -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) - -%description -Citus horizontally scales PostgreSQL across commodity servers -using sharding and replication. Its query engine parallelizes -incoming SQL queries across these servers to enable real-time -responses on large datasets. - -Citus extends the underlying database rather than forking it, -which gives developers and enterprises the power and familiarity -of a traditional relational database. As an extension, Citus -supports new PostgreSQL releases, allowing users to benefit from -new features while maintaining compatibility with existing -PostgreSQL tools. Note that Citus supports many (but not all) SQL -commands. - -%prep -%setup -q -n %{sname}-%{version} - -%build -%configure PG_CONFIG=%{pginstdir}/bin/pg_config --with-extra-version="%{?conf_extra_version}" --with-security-flags -make %{?_smp_mflags} - -%install -%make_install -# Install documentation with a better name: -%{__mkdir} -p %{buildroot}%{pginstdir}/doc/extension -%{__cp} README.md %{buildroot}%{pginstdir}/doc/extension/README-%{sname}.md -# Set paths to be packaged other than LICENSE, README & CHANGELOG.md -echo %{pginstdir}/include/server/citus_*.h >> installation_files.list -echo %{pginstdir}/include/server/distributed/*.h >> installation_files.list -echo %{pginstdir}/lib/%{sname}.so >> installation_files.list -echo %{pginstdir}/share/extension/%{sname}-*.sql >> installation_files.list -echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list -%ifarch ppc64 ppc64le - %else - %if 0%{?rhel} && 0%{?rhel} <= 6 - %else - echo %{pginstdir}/lib/bitcode/%{sname}/*.bc >> installation_files.list - echo %{pginstdir}/lib/bitcode/%{sname}*.bc >> installation_files.list - echo %{pginstdir}/lib/bitcode/%{sname}/*/*.bc >> installation_files.list - - # Columnar does not exist in Citus versions < 10.0 - # At this point, we don't have %{pginstdir}, - # so first check build directory for columnar. - [[ -d %{buildroot}%{pginstdir}/lib/bitcode/columnar/ ]] && echo %{pginstdir}/lib/bitcode/columnar/*.bc >> installation_files.list - %endif -%endif - -%clean -%{__rm} -rf %{buildroot} - -%files -f installation_files.list -%files -%defattr(-,root,root,-) -%doc CHANGELOG.md -%if 0%{?rhel} && 0%{?rhel} <= 6 -%doc LICENSE -%else -%license LICENSE -%endif -%doc %{pginstdir}/doc/extension/README-%{sname}.md - -%changelog -* Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 -- Update to Citus 10.0.3 - -* Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 -- Official 10.0.2 release of Citus - -* Wed Jan 27 2021 - gurkanindibay 9.5.2.citus-1 -- Official 9.5.2 release of Citus - -* Tue Jan 5 2021 - gurkanindibay 9.4.4.citus-1 -- Official 9.4.4 release of Citus - -* Wed Dec 2 2020 - Onur Tirtir 9.5.1.citus-1 -- Official 9.5.1 release of Citus - -* Tue Nov 24 2020 - Onur Tirtir 9.4.3.citus-1 -- Official 9.4.3 release of Citus - -* Wed Nov 11 2020 - Onur Tirtir 9.5.0.citus-1 -- Official 9.5.0 release of Citus - -* Thu Oct 22 2020 - Onur Tirtir 9.4.2.citus-1 -- Official 9.4.2 release of Citus - -* Wed Sep 30 2020 - Onur Tirtir 9.4.1.citus-1 -- Official 9.4.1 release of Citus - -* Tue Jul 28 2020 - Onur Tirtir 9.4.0.citus-1 -- Official 9.4.0 release of Citus - -* Mon Jul 27 2020 - Onur Tirtir 9.3.5.citus-1 -- Official 9.3.5 release of Citus - -* Wed Jul 22 2020 - Onur Tirtir 9.3.4.citus-1 -- Official 9.3.4 release of Citus - -* Mon Jul 13 2020 - Onur Tirtir 9.3.3.citus-1 -- Official 9.3.3 release of Citus - -* Thu May 7 2020 - Onur Tirtir 9.3.0.citus-1 -- Update to Citus 9.3.0 - -* Tue Mar 31 2020 - Onur Tirtir 9.2.4.citus-1 -- Update to Citus 9.2.4 - -* Thu Mar 26 2020 - Onur Tirtir 9.2.3.citus-1 -- Update to Citus 9.2.3 - -* Fri Mar 6 2020 - Onur Tirtir 9.0.2.citus-1 -- Update to Citus 9.0.2 - -* Fri Mar 6 2020 - Onur Tirtir 9.2.2.citus-1 -- Update to Citus 9.2.2 - -* Fri Feb 14 2020 - Onur Tirtir 9.2.1.citus-1 -- Update to Citus 9.2.1 - -* Mon Feb 10 2020 - Onur Tirtir 9.2.0.citus-1 -- Update to Citus 9.2.0 - -* Wed Dec 18 2019 - Onur Tirtir 9.1.1.citus-1 -- Update to Citus 9.1.1 - -* Thu Nov 28 2019 - Onur Tirtir 9.1.0.citus-1 -- Update to Citus 9.1.0 - -* Wed Oct 30 2019 - Hanefi Onaldi 9.0.1.citus-1 -- Update to Citus 9.0.1 - -* Thu Oct 10 2019 - Hanefi Onaldi 9.0.0.citus-1 -- Update to Citus 9.0.0 - -* Fri Aug 9 2019 - Hanefi Onaldi 8.3.2.citus-1 -- Update to Citus 8.3.2 - -* Mon Jul 29 2019 - Hanefi Onaldi 8.3.1.citus-1 -- Update to Citus 8.3.1 - -* Wed Jul 10 2019 - Burak Velioglu 8.3.0.citus-1 -- Update to Citus 8.3.0 - -* Wed Jun 12 2019 - Burak Velioglu 8.2.2.citus-1 -- Update to Citus 8.2.2 - -* Wed Apr 3 2019 - Burak Velioglu 8.2.1.citus-1 -- Update to Citus 8.2.1 - -* Wed Apr 3 2019 - Burak Velioglu 8.1.2.citus-1 -- Update to Citus 8.1.2 - -* Thu Mar 28 2019 - Burak Velioglu 8.2.0.citus-1 -- Update to Citus 8.2.0 - -* Wed Jan 9 2019 - Burak Velioglu 8.0.3.citus-1 -- Update to Citus 8.0.3 - -* Mon Jan 7 2019 - Burak Velioglu 8.1.1.citus-1 -- Update to Citus 8.1.1 - -* Tue Dec 18 2018 - Burak Velioglu 8.1.0.citus-1 -- Update to Citus 8.1.0 - -* Thu Dec 13 2018 - Burak Velioglu 8.0.2.citus-1 -- Update to Citus 8.0.2 - -* Wed Dec 12 2018 - Burak Velioglu 7.5.4.citus-1 -- Update to Citus 7.5.4 - -* Wed Nov 28 2018 - Burak Velioglu 8.0.1.citus-1 -- Update to Citus 8.0.1 - -* Wed Nov 28 2018 - Burak Velioglu 7.5.3.citus-1 -- Update to Citus 7.5.3 - -* Wed Nov 14 2018 - Burak Velioglu 7.5.2.citus-1 -- Update to Citus 7.5.2 - -* Fri Nov 02 2018 - Burak Velioglu 8.0.0.citus-1 -- Update to Citus 8.0.0 - -* Wed Aug 29 2018 - Burak Velioglu 7.5.1.citus-1 -- Update to Citus 7.5.1 - -* Fri Jul 27 2018 - Mehmet Furkan Sahin 7.4.2.citus-1 -- Update to Citus 7.4.2 - -* Wed Jul 25 2018 - Mehmet Furkan Sahin 7.5.0.citus-1 -- Update to Citus 7.5.0 - -* Wed Jun 20 2018 - Burak Velioglu 7.4.1.citus-1 -- Update to Citus 7.4.1 - -* Thu May 17 2018 - Burak Velioglu 7.2.2.citus-1 -- Update to Citus 7.2.2 - -* Tue May 15 2018 - Burak Velioglu 7.4.0.citus-1 -- Update to Citus 7.4.0 - -* Thu Mar 15 2018 - Burak Velioglu 7.3.0.citus-1 -- Update to Citus 7.3.0 - -* Tue Feb 6 2018 - Burak Velioglu 7.2.1.citus-1 -- Update to Citus 7.2.1 - -* Tue Jan 16 2018 - Burak Velioglu 7.2.0.citus-1 -- Update to Citus 7.2.0 - -* Thu Jan 11 2018 - Burak Velioglu 6.2.5.citus-1 -- Update to Citus 6.2.5 - -* Fri Jan 05 2018 - Burak Velioglu 7.1.2.citus-1 -- Update to Citus 7.1.2 - -* Tue Dec 05 2017 - Burak Velioglu 7.1.1.citus-1 -- Update to Citus 7.1.1 - -* Wed Nov 15 2017 - Burak Velioglu 7.1.0.citus-1 -- Update to Citus 7.1.0 - -* Mon Oct 16 2017 - Burak Yucesoy 7.0.3.citus-1 -- Update to Citus 7.0.3 - -* Thu Sep 28 2017 - Burak Yucesoy 7.0.2.citus-1 -- Update to Citus 7.0.2 - -* Thu Sep 28 2017 - Burak Yucesoy 6.2.4.citus-1 -- Update to Citus 6.2.4 - -* Thu Sep 28 2017 - Burak Yucesoy 6.1.3.citus-1 -- Update to Citus 6.1.3 - -* Tue Sep 12 2017 - Burak Yucesoy 7.0.1.citus-1 -- Update to Citus 7.0.1 - -* Tue Aug 29 2017 - Burak Yucesoy 7.0.0.citus-1 -- Update to Citus 7.0.0 - -* Thu Jul 13 2017 - Burak Yucesoy 6.2.3.citus-1 -- Update to Citus 6.2.3 - -* Wed Jun 7 2017 - Burak Velioglu 6.2.2.citus-1 -- Update to Citus 6.2.2 - -* Wed Jun 7 2017 - Jason Petersen 6.1.2.citus-1 -- Update to Citus 6.1.2 - -* Wed May 24 2017 - Jason Petersen 6.2.1.citus-1 -- Update to Citus 6.2.1 - -* Tue May 16 2017 - Burak Yucesoy 6.2.0.citus-1 -- Update to Citus 6.2.0 - -* Fri May 5 2017 - Metin Doslu 6.1.1.citus-1 -- Update to Citus 6.1.1 - -* Thu Feb 9 2017 - Burak Yucesoy 6.1.0.citus-1 -- Update to Citus 6.1.0 - -* Wed Feb 8 2017 - Jason Petersen 6.0.1.citus-2 -- Transitional package to guide users to new package name - -* Wed Nov 30 2016 - Burak Yucesoy 6.0.1.citus-1 -- Update to Citus 6.0.1 - -* Tue Nov 8 2016 - Jason Petersen 6.0.0.citus-1 -- Update to Citus 6.0.0 - -* Tue Nov 8 2016 - Jason Petersen 5.2.2.citus-1 -- Update to Citus 5.2.2 - -* Tue Sep 6 2016 - Jason Petersen 5.2.1.citus-1 -- Update to Citus 5.2.1 - -* Wed Aug 17 2016 - Jason Petersen 5.2.0.citus-1 -- Update to Citus 5.2.0 - -* Mon Aug 1 2016 - Jason Petersen 5.2.0-0.1.rc.1 -- Release candidate for 5.2 - -* Fri Jun 17 2016 - Jason Petersen 5.1.1-1 -- Update to Citus 5.1.1 - -* Tue May 17 2016 - Jason Petersen 5.1.0-1 -- Update to Citus 5.1.0 - -* Mon May 16 2016 - Jason Petersen 5.1.0-0.2.rc.2 -- Fix EXPLAIN output when FORMAT JSON in use - -* Wed May 4 2016 - Jason Petersen 5.1.0-0.1.rc.1 -- Release candidate for 5.1 - -* Fri Apr 15 2016 - Jason Petersen 5.0.1-1 -- Fixes issues on 32-bit systems - -* Fri Mar 25 2016 - Devrim Gündüz 5.0.0-1 -- Initial RPM packaging for PostgreSQL RPM Repository, - based on the spec file of Jason Petersen @ Citus. From 6fa30fe7d2df57dc47c43e8f410053d7a433f5e3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 2 Apr 2021 22:49:22 +0300 Subject: [PATCH 028/106] Change pipeline name Add trigger to run on all branches and pr requests --- .github/workflows/tool-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 5086d15c..4dcc1711 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -1,6 +1,6 @@ # This is a basic workflow to help you get started with Actions -name: CI +name: Tool Tests env: GH_TOKEN: ${{ secrets.GH_TOKEN }} @@ -10,9 +10,9 @@ env: on: # Triggers the workflow on push or pull request events but only for the develop branch push: - branches: [ develop ] + branches: "**" pull_request: - branches: [ develop ] + branches: "**" # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From eaa0484ddfcf778261abb204fb8dec30e0688491 Mon Sep 17 00:00:00 2001 From: Gurkan Indibay Date: Sat, 3 Apr 2021 04:28:15 +0000 Subject: [PATCH 029/106] Add pycache files into gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5127c413..1189a22b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ # generated man pages *.1 +/python/__pycache__/ +/python/tests/__pycache__/ From 094699daad40bca4a254f91e64aa0c0c1bbe878a Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 7 Apr 2021 17:29:39 +0300 Subject: [PATCH 030/106] Add prepare release script --- bash/prepare_release.sh | 0 python/common_tool_methods.py | 123 +++ python/common_validations.py | 28 + python/prepare_release.py | 203 ++++ python/test-features.py | 10 + python/tests/projects/citus/configure.in | 312 ++++++ .../src/backend/distributed/citus.control | 6 + .../test/regress/expected/multi_extension.out | 911 ++++++++++++++++++ .../src/test/regress/sql/multi_extension.sql | 498 ++++++++++ python/tests/test_common_tool_methods.py | 81 ++ python/tests/test_prepare_release.py | 28 + .../tests/test_update_package_properties.py | 9 - python/update_package_properties.py | 97 +- 13 files changed, 2212 insertions(+), 94 deletions(-) create mode 100644 bash/prepare_release.sh create mode 100644 python/common_tool_methods.py create mode 100644 python/common_validations.py create mode 100644 python/prepare_release.py create mode 100644 python/test-features.py create mode 100644 python/tests/projects/citus/configure.in create mode 100644 python/tests/projects/citus/src/backend/distributed/citus.control create mode 100644 python/tests/projects/citus/src/test/regress/expected/multi_extension.out create mode 100644 python/tests/projects/citus/src/test/regress/sql/multi_extension.sql create mode 100644 python/tests/test_common_tool_methods.py create mode 100644 python/tests/test_prepare_release.py diff --git a/bash/prepare_release.sh b/bash/prepare_release.sh new file mode 100644 index 00000000..e69de29b diff --git a/python/common_tool_methods.py b/python/common_tool_methods.py new file mode 100644 index 00000000..a343ba84 --- /dev/null +++ b/python/common_tool_methods.py @@ -0,0 +1,123 @@ +import re +import subprocess +from datetime import datetime +from typing import Dict, List + +from github import Repository, PullRequest +from jinja2 import Environment, FileSystemLoader + +from .common_validations import * + + +def get_spec_file_name(project_name: str) -> str: + return f"{project_name}.spec" + + +def get_version_number(version: str, fancy: bool, fancy_release_count: int) -> str: + fancy_suffix = f"-{fancy_release_count}" if fancy else "" + return f"{version}{fancy_suffix}" + + +def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, fancy_release_count: int) -> str: + fancy_suffix = f"-{fancy_release_count}" if fancy else "" + return f"{version}.{project_name}{fancy_suffix}" + + +def get_template_environment(template_dir: str) -> Environment: + file_loader = FileSystemLoader(template_dir) + env = Environment(loader=file_loader) + return env + + +def find_nth_overlapping(subject_string, search_string, n) -> int: + start = subject_string.find(search_string) + + while start >= 0 and n > 1: + start = subject_string.find(search_string, start + 1) + n -= 1 + return start + + +def find_nth_overlapping_line_by_regex(subject_string, regex_pattern, n) -> int: + lines = subject_string.splitlines() + counter = 0 + index = -1 + for i in range(len(lines)): + if re.match(regex_pattern, lines[i]): + counter = counter + 1 + if counter == n: + index = i + break + return index + + +def remove_string_inside_parentheses(param: str) -> str: + return re.sub(r"[(\[].*?[)\]]", "", param) + + +def run(command, *args, **kwargs): + result = subprocess.run(command, *args, check=True, shell=True, **kwargs) + return result + + +def cherry_pick_prs(prs: List[PullRequest.PullRequest]): + for pr in prs: + commits = pr.get_commits() + for single_commit in commits: + cp_result = run(f"git cherry-pick {single_commit.commit.sha}") + print(f"Cherry pick result for PR No {pr.number} and commit sha {single_commit.commit.sha}: {cp_result} ") + + +def get_version_details(version: is_version(str)) -> Dict[str, str]: + version_parts = version.split(".") + return {"major": version_parts[0], "minor": version_parts[1], "patch": version_parts[2]} + + +def is_major_release(version: is_version(str)) -> bool: + version_info = get_version_details(version) + return version_info["patch"] == "0" + + +def str_array_to_str(str_array: List[str]) -> str: + result_str = "" + for i in range(len(str_array)): + result_str = result_str + str_array[i] + "\n" + return result_str + + +def get_prs(repo: Repository.Repository, earliest_date: datetime, base_branch: str, last_date: datetime = None): + pulls = repo.get_pulls(state="all", base=base_branch, sort="created", direction="desc") + picked_pulls = [] + for pull in pulls: + if not pull.is_merged(): + continue + if last_date is not None and pull.created_at > last_date: + continue + if pull.created_at < earliest_date: + break + if pull.merged_at is not None and pull.merged_at > earliest_date: + picked_pulls.append(pull) + return picked_pulls + + +def get_pr_issues_by_label(prs: List[PullRequest.PullRequest], label_name: str): + filtered_prs = [] + for pr in prs: + filtered_labels = filter(lambda label: label.name == label_name, pr.labels) + if len(list(filtered_labels)) > 0: + filtered_prs.append(pr) + return filtered_prs + + +def replace_line_in_file(file: str, match_regex: str, replace_str: str): + with open(file, "r") as reader: + file_content = reader.read() + lines = file_content.splitlines() + line_counter = 0 + for line in lines: + if re.match(match_regex, line): + lines[line_counter] = replace_str + line_counter = line_counter + 1 + edited_content = str_array_to_str(lines) + with open(file, "w") as writer: + writer.write(edited_content) diff --git a/python/common_validations.py b/python/common_validations.py new file mode 100644 index 00000000..9b47805c --- /dev/null +++ b/python/common_validations.py @@ -0,0 +1,28 @@ +import string_utils +from parameters_validation import parameter_validation +import re + + +@parameter_validation +def is_version(version: str): + if version is None or not version: + raise ValueError("version should be non-empty and should not be None") + if not re.match(r"\d+\.\d+\.\d+$", version): + raise ValueError( + "version should include 3 levels of versions consists of numbers separated with dots. e.g: 10.0.1") + + +@parameter_validation +def is_tag(tag: str): + if tag is None or not tag: + raise ValueError("tag should be non-empty and should not be None") + if not re.match(r"v\d+\.\d+\.\d+$", tag): + raise ValueError( + "tag should start with v and should include 3 levels of versions consists of numbers " + + "separated with dots. e.g: v10.0.1") + + +@parameter_validation +def is_email(email: str): + if not string_utils.is_email(email): + raise ValueError("Parameter is not in email format") diff --git a/python/prepare_release.py b/python/prepare_release.py new file mode 100644 index 00000000..ef03fea6 --- /dev/null +++ b/python/prepare_release.py @@ -0,0 +1,203 @@ +import os + +from github import Github + +from .common_tool_methods import * + +MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" +CITUS_CONTROL = "src/backend/distributed/citus.control" +MULTI_EXTENSION_OUT = "src/test/regress/expected/multi_extension.out" +CONFIG_PY = "src/test/regress/upgrade/config.py" +DISTRIBUTED_DIR_PATH = "src/backend/distributed" +CONFIGURE_IN = "configure.in" +CITUS_CONTROL_SEARCH_PATTERN = r"^default_version*" + +MULT_EXT_SEARCH_PATTERN = r"^\d{1,2}\.\d{1,2}devel$" +CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" +REPO_OWNER = "citusdata" + + +def update_release(github_token: str, project_name: str, project_version: is_version(str), main_branch: str, + earliest_pr_date: datetime, exec_path: str): + multi_extension_sql_path = f"{exec_path}/{MULTI_EXTENSION_SQL}" + citus_control_file_path = f"{exec_path}/{CITUS_CONTROL}" + multi_extension_out_path = f"{exec_path}/{MULTI_EXTENSION_OUT}" + configure_in_path = f"{exec_path}/{CONFIGURE_IN}" + config_py_path = f"{exec_path}/{CONFIG_PY}" + distributed_dir_path = f"{exec_path}/{DISTRIBUTED_DIR_PATH}" + + project_version_details = get_version_details(project_version) + default_upcoming_version = f'{project_version_details["major"]}.{project_version_details["minor"]}.' \ + f'{str(int(project_version_details["patch"]) + 1)}' + upcoming_version = default_upcoming_version if os.getenv("UPCOMING_VERSION") is None else os.getenv( + "UPCOMING_VERSION") + upcoming_version_details = get_version_details(upcoming_version) + upcoming_minor_version = f'{upcoming_version_details["major"]}.{upcoming_version_details["minor"]}' + release_branch_name = f'release-{project_version_details["major"]}.{project_version_details["minor"]}' + g = Github(github_token) + repository = g.get_repo(f"{REPO_OWNER}/{project_name}") + + # if major release + if is_major_release(project_version): + print(f"{project_version} is a major release. Executing Major release flow ") + # create release-X-Y branch + print(f"Preparing {release_branch_name}...") + # checkout master + print(f"Checking out {main_branch}...") + run(f"git checkout {main_branch}") + run(f"git pull") + print(f"OK {main_branch} checked out and pulled") + # create release branch in release-X.Y format + print(f"Creating release branch with name {release_branch_name}...") + run(f'git checkout -b {release_branch_name}') + print(f"OK {release_branch_name} created") + # change version info in configure.in file + print(f"Updating version on file {configure_in_path}...") + replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{project_version}])") + print(f"OK {configure_in_path} file is updated with project version {project_version}.") + # update_configure_in(project_version) + + # execute "autoconf -f" + print(f"Executing autoconf -f command...") + run("autoconf -f") + print(f"OK autoconf -f executed.") + # change version info in multi_extension.out + print(f"Updating version on file {multi_extension_out_path}...") + replace_line_in_file(multi_extension_out_path, MULT_EXT_SEARCH_PATTERN, + f"{project_version}") + print(f"OK {multi_extension_out_path} file is updated with project version {project_version}.") + # commit all changes + print(f"Committing changes for branch {release_branch_name}...") + run(f' git commit -am "Bump {project_name} version to {project_version} "') + print(f"OK Changes committed for {release_branch_name}") + # push release branch (No PR creation!!!) + print(f"Pushing changes for {release_branch_name} into remote origin...") + run(f"git push --set-upstream origin {release_branch_name}") + print(f"OK Changes pushed for {release_branch_name}") + print(f"OK {release_branch_name} prepared.") + # Increase version number and change upcoming version + upcoming_version_branch = f"master-update-version {datetime.now().timestamp()}" + print(f"Preparing upcoming version branch for the new master with name {upcoming_version_branch}...") + # checkout master + print(f"Checking out {main_branch}...") + run(f"git checkout {main_branch}") + print(f"OK {main_branch} checked out.") + # create master-update-version-$curtime branch + print(f"Checking out {upcoming_version_branch}...") + run(f"git checkout -b {upcoming_version_branch}") + print(f"{upcoming_version_branch} checked out.") + # update version info with upcoming version on configure.in + print(f"Updating {configure_in_path} file with the upcoming version {upcoming_version}...") + replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{upcoming_version}])") + print(f"U{configure_in_path} file updated with the upcoming version {upcoming_version}...") + # update version info with upcoming version on config.py + print(f"Updating {config_py_path} file with the upcoming version {upcoming_minor_version}...") + replace_line_in_file(config_py_path, "^MASTER_VERSION =*", + f'MASTER_VERSION = {upcoming_minor_version}') + print(f"U{config_py_path} file updated with the upcoming version {upcoming_minor_version}...") + # execute autoconf -f + print(f"Executing autoconf -f command...") + run("autoconf -f") + print(f"OK autoconf -f executed.") + # update version info with upcoming version on multiextension.out + # TODO May add a special version descriptor to address lines directly to be replaced + print(f"Updating {multi_extension_out_path} file with the upcoming version {upcoming_version}...") + replace_line_in_file(multi_extension_out_path, MULT_EXT_SEARCH_PATTERN, + f"{upcoming_version}") + print(f"U{multi_extension_out_path} file updated with the upcoming version {upcoming_version}...") + # get current schema version from citus.control + print(f"Reading current schema version from {citus_control_file_path}...") + current_schema_version = "" + with open(citus_control_file_path, "r") as cc_reader: + cc_file_content = cc_reader.read() + cc_lines = cc_file_content.splitlines() + for cc_line in cc_lines: + if re.match(CITUS_CONTROL_SEARCH_PATTERN, cc_line): + current_schema_version = cc_line + + if len(current_schema_version) == 0: + raise ValueError("Version info could not be found in citus.control file") + + print(f"OK Current schema version is {current_schema_version}") + + # find current schema version info and update it with upcoming version in multi_extension.sql file + print( + f"Updating schema version {current_schema_version} on {multi_extension_sql_path} " + f"file with the upcoming version {upcoming_version}...") + replace_line_in_file(multi_extension_sql_path, + f"ALTER EXTENSION citus UPDATE TO {current_schema_version}", + f"ALTER EXTENSION citus UPDATE TO {upcoming_minor_version}") + print(f"OK Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}") + # find current schema version info and update it with upcoming version in multi_extension.out file + print( + f"Updating schema version {current_schema_version} on {multi_extension_out_path} " + f"file with the upcoming version {upcoming_version}...") + replace_line_in_file(multi_extension_out_path, + f"ALTER EXTENSION citus UPDATE TO {current_schema_version}", + f"ALTER EXTENSION citus UPDATE TO {upcoming_minor_version}") + print(f"OK Current schema version updated on {multi_extension_out_path} to {upcoming_minor_version}") + # create new sql file with the name ">./src/backend/distributed/citus--$current_schema_version-- + # $upcoming_minor_version-1.sql" + newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" + print(f"Creating file {newly_created_sql_file}...") + with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: + content = f"/* citus--{current_schema_version}--{upcoming_minor_version}-1 */" + content = content + "\n\n" + content = content + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" + f_writer.write(content) + print(f"{newly_created_sql_file} created.") + # change version in citus.control file + print(f"Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}...") + replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, + f"default_version={upcoming_minor_version}-1") + print(f"U{citus_control_file_path} file updated with the upcoming version {upcoming_minor_version}...") + # commit and push changes on master-update-version-$curtime branch + print(f"Committing changes for branch {upcoming_version_branch}...") + run(f'git commit -am "{project_name} version to {project_version}"') + print(f"OK Changes committed for {upcoming_version_branch}") + print(f"Pushing changes for {upcoming_version_branch} into remote origin...") + run(f"git push --set-upstream origin {upcoming_version_branch}") + print(f"OK Changes pushed for {upcoming_version_branch}") + # create pull request + print(f"Creating pull request for {upcoming_version_branch}") + pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, + head=upcoming_version_branch, body="") + print(f"OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}") + else: + print(f"{project_version} is a patch release. Executing Patch release flow ") + # checkout release branch in release-X.Y format + print(f"Checking out {release_branch_name}...") + run(f'git checkout {release_branch_name}') + print(f"{release_branch_name} checked out") + # list all pr's with backport labels + print(f"Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}") + all_related_prs = get_prs(repository, earliest_pr_date, main_branch) + # get commits for selected prs with backport label + prs_with_backport = get_pr_issues_by_label(all_related_prs, "backport") + print(f"OK {len(prs_with_backport)} PRs with backport label found. PR list is as below") + for pr in prs_with_backport: + print(f"No:{pr.number} Title:{pr.title}\n") + + # cherrypick all commits with backport label + + print(f"Cherry-picking PRs on {release_branch_name}...") + cherry_pick_prs(prs_with_backport) + print(f"OK Cherry pick completed for all PRs on branch {release_branch_name}") + # change version info in configure.in file + + replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{project_version}])") + # execute "auto-conf " (Not with f flag!!!) + run("autoconf") + # change version info in multi_extension.out + print(f"Updating {multi_extension_out_path} file with the project version {project_version}...") + replace_line_in_file(multi_extension_out_path, MULT_EXT_SEARCH_PATTERN, + f"{project_version}") + print(f"OK {multi_extension_out_path} file is updated with project version {project_version}.") + # commit all-changes + print(f"Committing changes for branch {release_branch_name}...") + run(f'git commit -am "{project_name} version to {project_version}"') + print(f"OK Changes committed for {release_branch_name}") + # create and push release-$minor_version-push-$curTime branch + print(f"Pushing changes for {release_branch_name} into remote origin...") + run(f"git push --set-upstream origin {release_branch_name}") + print(f"OK Changes pushed for {release_branch_name}") diff --git a/python/test-features.py b/python/test-features.py new file mode 100644 index 00000000..aa619943 --- /dev/null +++ b/python/test-features.py @@ -0,0 +1,10 @@ +from github import Github, Repository, PullRequest +from datetime import datetime + +from typing import List +import subprocess +import uuid + +g = Github("6df6f95934896fd559e4b2f7eb630c49f8dad560") +repository = g.get_repo(f"citusdata/citus") + diff --git a/python/tests/projects/citus/configure.in b/python/tests/projects/citus/configure.in new file mode 100644 index 00000000..4b2e1d6d --- /dev/null +++ b/python/tests/projects/citus/configure.in @@ -0,0 +1,312 @@ +# Citus autoconf input script. +# +# Converted into an actual configure script by autogen.sh. This +# conversion only has to be done when configure.in changes. To avoid +# everyone needing autoconf installed, the resulting files are checked +# into the SCM. + +AC_INIT([Citus], [10.1devel]) +AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) + +# we'll need sed and awk for some of the version commands +AC_PROG_SED +AC_PROG_AWK + +# CITUS_NAME definition +AC_DEFINE_UNQUOTED(CITUS_NAME, "$PACKAGE_NAME", [Citus full name as a string]) + +case $PACKAGE_NAME in + 'Citus Enterprise') citus_edition=enterprise ;; + Citus) citus_edition=community ;; + *) AC_MSG_ERROR([Unrecognized package name.]) ;; +esac + +# CITUS_EDITION definition +AC_DEFINE_UNQUOTED(CITUS_EDITION, "$citus_edition", [Citus edition as a string]) + +# CITUS_MAJORVERSION definition +[CITUS_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\.[0-9][0-9]*\)'`] +AC_DEFINE_UNQUOTED(CITUS_MAJORVERSION, "$CITUS_MAJORVERSION", [Citus major version as a string]) + +# CITUS_VERSION definition +PGAC_ARG_REQ(with, extra-version, [STRING], [append STRING to version], + [CITUS_VERSION="$PACKAGE_VERSION$withval"], + [CITUS_VERSION="$PACKAGE_VERSION"]) +AC_DEFINE_UNQUOTED(CITUS_VERSION, "$CITUS_VERSION", [Citus version as a string]) + +# CITUS_VERSION_NUM definition +# awk -F is a regex on some platforms, and not on others, so make "." a tab +[CITUS_VERSION_NUM="`echo "$PACKAGE_VERSION" | sed 's/[A-Za-z].*$//' | +tr '.' ' ' | +$AWK '{printf "%d%02d%02d", $1, $2, (NF >= 3) ? $3 : 0}'`"] +AC_DEFINE_UNQUOTED(CITUS_VERSION_NUM, $CITUS_VERSION_NUM, [Citus version as a number]) + +# CITUS_EXTENSIONVERSION definition +[CITUS_EXTENSIONVERSION="`grep '^default_version' $srcdir/src/backend/distributed/citus.control | cut -d\' -f2`"] +AC_DEFINE_UNQUOTED([CITUS_EXTENSIONVERSION], "$CITUS_EXTENSIONVERSION", [Extension version expected by this Citus build]) + +# Re-check for flex. That allows to compile citus against a postgres +# which was built without flex available (possible because generated +# files are included) +AC_PATH_PROG([FLEX], [flex]) + +# Locate pg_config binary +AC_ARG_VAR([PG_CONFIG], [Location to find pg_config for target PostgreSQL instalation (default PATH)]) +AC_ARG_VAR([PATH], [PATH for target PostgreSQL install pg_config]) + +if test -z "$PG_CONFIG"; then + AC_PATH_PROG(PG_CONFIG, pg_config) +fi + +if test -z "$PG_CONFIG"; then + AC_MSG_ERROR([Could not find pg_config. Set PG_CONFIG or PATH.]) +fi + +# check we're building against a supported version of PostgreSQL +citusac_pg_config_version=$($PG_CONFIG --version 2>/dev/null) +version_num=$(echo "$citusac_pg_config_version"| + $SED -e 's/^PostgreSQL \([[0-9]]*\)\(\.[[0-9]]*\)\{0,1\}\(.*\)$/\1\2/') + +# if PostgreSQL version starts with two digits, the major version is those digits +version_num=$(echo "$version_num"| $SED -e 's/^\([[0-9]]\{2\}\)\(.*\)$/\1/') + +if test -z "$version_num"; then + AC_MSG_ERROR([Could not detect PostgreSQL version from pg_config.]) +fi + +if test "$version_num" != '12' -a "$version_num" != '13'; then + AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) +else + AC_MSG_NOTICE([building against PostgreSQL $version_num]) +fi; + +# Check whether we're building inside the source tree, if not, prepare +# the build directory. +if test "$srcdir" -ef '.' ; then + vpath_build=no +else + vpath_build=yes + _AS_ECHO_N([preparing build tree... ]) + citusac_abs_top_srcdir=`cd "$srcdir" && pwd` + $SHELL "$citusac_abs_top_srcdir/prep_buildtree" "$citusac_abs_top_srcdir" "." \ + || AC_MSG_ERROR(failed) + AC_MSG_RESULT(done) +fi +AC_SUBST(vpath_build) + +# Allow to overwrite the C compiler, default to the one postgres was +# compiled with. We don't want autoconf's default CFLAGS though, so save +# those. +SAVE_CFLAGS="$CFLAGS" +AC_PROG_CC([$($PG_CONFIG --cc)]) +CFLAGS="$SAVE_CFLAGS" + +host_guess=`${SHELL} $srcdir/config/config.guess` + +# Create compiler version string +if test x"$GCC" = x"yes" ; then + cc_string=`${CC} --version | sed q` + case $cc_string in [[A-Za-z]]*) ;; *) cc_string="GCC $cc_string";; esac +elif test x"$SUN_STUDIO_CC" = x"yes" ; then + cc_string=`${CC} -V 2>&1 | sed q` +else + cc_string=$CC +fi + +AC_CHECK_SIZEOF([void *]) + +AC_DEFINE_UNQUOTED(CITUS_VERSION_STR, + ["$PACKAGE_NAME $CITUS_VERSION on $host_guess, compiled by $cc_string, `expr $ac_cv_sizeof_void_p \* 8`-bit"], + [A string containing the version number, platform, and C compiler]) + +# Locate source and build directory of the postgres we're building +# against. Can't rely on either still being present, but e.g. optional +# test infrastructure can rely on it. +POSTGRES_SRCDIR=$(grep ^abs_top_srcdir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) +POSTGRES_BUILDDIR=$(grep ^abs_top_builddir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) + + +# check for a number of CFLAGS that make development easier + +# CITUSAC_PROG_CC_CFLAGS_OPT +# ----------------------- +# Given a string, check if the compiler supports the string as a +# command-line option. If it does, add the string to CFLAGS. +AC_DEFUN([CITUSAC_PROG_CC_CFLAGS_OPT], +[define([Ac_cachevar], [AS_TR_SH([citusac_cv_prog_cc_cflags_$1])])dnl +AC_CACHE_CHECK([whether $CC supports $1], [Ac_cachevar], +[citusac_save_CFLAGS=$CFLAGS +flag=$1 +case $flag in -Wno*) + flag=-W$(echo $flag | cut -c 6-) +esac +CFLAGS="$citusac_save_CFLAGS $flag" +ac_save_c_werror_flag=$ac_c_werror_flag +ac_c_werror_flag=yes +_AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [Ac_cachevar=yes], + [Ac_cachevar=no]) +ac_c_werror_flag=$ac_save_c_werror_flag +CFLAGS="$citusac_save_CFLAGS"]) +if test x"$Ac_cachevar" = x"yes"; then + CITUS_CFLAGS="$CITUS_CFLAGS $1" +fi +undefine([Ac_cachevar])dnl +])# CITUSAC_PROG_CC_CFLAGS_OPT + +CITUSAC_PROG_CC_CFLAGS_OPT([-std=gnu99]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wall]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wextra]) +# disarm options included in the above, which are too noisy for now +CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-unused-parameter]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-sign-compare]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-missing-field-initializers]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-clobbered]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-gnu-variable-sized-type-not-at-end]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-declaration-after-statement]) +# And add a few extra warnings +CITUSAC_PROG_CC_CFLAGS_OPT([-Wendif-labels]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-format-attribute]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-declarations]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-prototypes]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wshadow]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=vla]) # visual studio does not support these +CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=implicit-int]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=implicit-function-declaration]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=return-type]) +# Security flags +# Flags taken from: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10203#guide +# We do not enforce the following flag because it is only available on GCC>=8 +CITUSAC_PROG_CC_CFLAGS_OPT([-fstack-clash-protection]) + +# +# --enable-coverage enables generation of code coverage metrics with gcov +# +AC_ARG_ENABLE([coverage], AS_HELP_STRING([--enable-coverage], [build with coverage testing instrumentation])) +if test "$enable_coverage" = yes; then + CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage" + CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG" + CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage" +fi + +# +# libcurl +# +PGAC_ARG_BOOL(with, libcurl, yes, + [do not use libcurl for anonymous statistics collection], + [AC_DEFINE([HAVE_LIBCURL], 1, [Define to 1 to build with libcurl support. (--with-libcurl)])]) + +if test "$with_libcurl" = yes; then + AC_CHECK_LIB(curl, curl_global_init, [], + [AC_MSG_ERROR([libcurl not found +If you have libcurl already installed, see config.log for details on the +failure. It is possible the compiler isn't looking in the proper directory. +Use --without-libcurl to disable anonymous statistics collection.])]) + AC_CHECK_HEADER(curl/curl.h, [], [AC_MSG_ERROR([libcurl header not found +If you have libcurl already installed, see config.log for details on the +failure. It is possible the compiler isn't looking in the proper directory. +Use --without-libcurl to disable libcurl support.])]) +fi + +# REPORTS_BASE_URL definition +PGAC_ARG_REQ(with, reports-hostname, [HOSTNAME], + [Use HOSTNAME as hostname for statistics collection and update checks], + [REPORTS_BASE_URL="https://${withval}"], + [REPORTS_BASE_URL="https://reports.citusdata.com"]) +AC_DEFINE_UNQUOTED(REPORTS_BASE_URL, "$REPORTS_BASE_URL", + [Base URL for statistics collection and update checks]) + +HAS_TABLEAM=yes +AC_DEFINE([HAS_TABLEAM], 1, [Define to 1 to build with table access method support, pg12 and up]) + +# Require lz4 & zstd only if we are compiling columnar +if test "$HAS_TABLEAM" == 'yes'; then + # + # LZ4 + # + PGAC_ARG_BOOL(with, lz4, yes, + [do not use lz4]) + AC_SUBST(with_lz4) + + if test "$with_lz4" = yes; then + AC_CHECK_LIB(lz4, LZ4_compress_default, [], + [AC_MSG_ERROR([lz4 library not found + If you have lz4 installed, see config.log for details on the + failure. It is possible the compiler isn't looking in the proper directory. + Use --without-lz4 to disable lz4 support.])]) + AC_CHECK_HEADER(lz4.h, [], [AC_MSG_ERROR([lz4 header not found + If you have lz4 already installed, see config.log for details on the + failure. It is possible the compiler isn't looking in the proper directory. + Use --without-lz4 to disable lz4 support.])]) + fi + + # + # ZSTD + # + PGAC_ARG_BOOL(with, zstd, yes, + [do not use zstd]) + AC_SUBST(with_zstd) + + if test "$with_zstd" = yes; then + AC_CHECK_LIB(zstd, ZSTD_decompress, [], + [AC_MSG_ERROR([zstd library not found + If you have zstd installed, see config.log for details on the + failure. It is possible the compiler isn't looking in the proper directory. + Use --without-zstd to disable zstd support.])]) + AC_CHECK_HEADER(zstd.h, [], [AC_MSG_ERROR([zstd header not found + If you have zstd already installed, see config.log for details on the + failure. It is possible the compiler isn't looking in the proper directory. + Use --without-zstd to disable zstd support.])]) + fi + +fi # test "$HAS_TABLEAM" == 'yes' + + +PGAC_ARG_BOOL(with, security-flags, no, + [use security flags]) +AC_SUBST(with_security_flags) + +if test "$with_security_flags" = yes; then +# Flags taken from: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10203#guide + +# We always want to have some compiler flags for security concerns. +SECURITY_CFLAGS="-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2 -z noexecstack -fpic -shared -Wl,-z,relro -Wl,-z,now -Wformat -Wformat-security -Werror=format-security" +CITUS_CFLAGS="$CITUS_CFLAGS $SECURITY_CFLAGS" +AC_MSG_NOTICE([Blindly added security flags for linker: $SECURITY_CFLAGS]) + +# We always want to have some clang flags for security concerns. +# This doesn't include "-Wl,-z,relro -Wl,-z,now" on purpuse, because bitcode is not linked. +# This doesn't include -fsanitize=cfi because it breaks builds on many distros including +# Debian/Buster, Debian/Stretch, Ubuntu/Bionic, Ubuntu/Xenial and EL7. +SECURITY_BITCODE_CFLAGS="-fsanitize=safe-stack -fstack-protector-strong -flto -fPIC -Wformat -Wformat-security -Werror=format-security" +CITUS_BITCODE_CFLAGS="$CITUS_BITCODE_CFLAGS $SECURITY_BITCODE_CFLAGS" +AC_MSG_NOTICE([Blindly added security flags for llvm: $SECURITY_BITCODE_CFLAGS]) + +AC_MSG_WARN([If you run into issues during linking or bitcode compilation, you can use --without-security-flags.]) +fi + +# Check if git is installed, when installed the gitref of the checkout will be baked in the application +AC_PATH_PROG(GIT_BIN, git) +AC_CHECK_FILE(.git,[HAS_DOTGIT=yes], [HAS_DOTGIT=]) + +AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS") +AC_SUBST(CITUS_BITCODE_CFLAGS, "$CITUS_BITCODE_CFLAGS") +AC_SUBST(CITUS_CPPFLAGS, "$CITUS_CPPFLAGS") +AC_SUBST(CITUS_LDFLAGS, "$LIBS $CITUS_LDFLAGS") +AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR") +AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR") +AC_SUBST(HAS_DOTGIT, "$HAS_DOTGIT") +AC_SUBST(HAS_TABLEAM, "$HAS_TABLEAM") + +AC_CONFIG_FILES([Makefile.global]) +AC_CONFIG_HEADERS([src/include/citus_config.h] [src/include/citus_version.h]) +AH_TOP([ +/* + * citus_config.h.in is generated by autoconf/autoheader and + * converted into citus_config.h by configure. Include when code needs to + * depend on determinations made by configure. + * + * Do not manually edit! + */ +]) +AC_OUTPUT diff --git a/python/tests/projects/citus/src/backend/distributed/citus.control b/python/tests/projects/citus/src/backend/distributed/citus.control new file mode 100644 index 00000000..a004b145 --- /dev/null +++ b/python/tests/projects/citus/src/backend/distributed/citus.control @@ -0,0 +1,6 @@ +# Citus extension +comment = 'Citus distributed database' +default_version = '10.1-1' +module_pathname = '$libdir/citus' +relocatable = false +schema = pg_catalog diff --git a/python/tests/projects/citus/src/test/regress/expected/multi_extension.out b/python/tests/projects/citus/src/test/regress/expected/multi_extension.out new file mode 100644 index 00000000..72256f54 --- /dev/null +++ b/python/tests/projects/citus/src/test/regress/expected/multi_extension.out @@ -0,0 +1,911 @@ +-- +-- MULTI_EXTENSION +-- +-- Tests around extension creation / upgrades +-- +-- It'd be nice to script generation of this file, but alas, that's +-- not done yet. +-- differentiate the output file for pg11 and versions above, with regards to objects +-- created per citus version depending on the postgres version. Upgrade tests verify the +-- objects are added in citus_finish_pg_upgrade() +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS version_above_eleven; + version_above_eleven +--------------------------------------------------------------------- + t +(1 row) + +SET citus.next_shard_id TO 580000; +SELECT $definition$ +CREATE OR REPLACE FUNCTION test.maintenance_worker() + RETURNS pg_stat_activity + LANGUAGE plpgsql +AS $$ +DECLARE + activity record; +BEGIN + DO 'BEGIN END'; -- Force maintenance daemon to start + -- we don't want to wait forever; loop will exit after 20 seconds + FOR i IN 1 .. 200 LOOP + PERFORM pg_stat_clear_snapshot(); + SELECT * INTO activity FROM pg_stat_activity + WHERE application_name = 'Citus Maintenance Daemon' AND datname = current_database(); + IF activity.pid IS NOT NULL THEN + RETURN activity; + ELSE + PERFORM pg_sleep(0.1); + END IF ; + END LOOP; + -- fail if we reach the end of this loop + raise 'Waited too long for maintenance daemon to start'; +END; +$$; +$definition$ create_function_test_maintenance_worker +\gset +CREATE TABLE prev_objects(description text); +CREATE TABLE extension_diff(previous_object text COLLATE "C", + current_object text COLLATE "C"); +CREATE FUNCTION print_extension_changes() +RETURNS TABLE(previous_object text, current_object text) +AS $func$ +BEGIN + TRUNCATE TABLE extension_diff; + + CREATE TABLE current_objects AS + SELECT pg_catalog.pg_describe_object(classid, objid, 0) AS description + FROM pg_catalog.pg_depend, pg_catalog.pg_extension e + WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass + AND refobjid = e.oid + AND deptype = 'e' + AND e.extname='citus'; + + INSERT INTO extension_diff + SELECT p.description previous_object, c.description current_object + FROM current_objects c FULL JOIN prev_objects p + ON p.description = c.description + WHERE p.description is null OR c.description is null; + + DROP TABLE prev_objects; + ALTER TABLE current_objects RENAME TO prev_objects; + + RETURN QUERY SELECT * FROM extension_diff ORDER BY 1, 2; +END +$func$ LANGUAGE plpgsql; +CREATE SCHEMA test; +:create_function_test_maintenance_worker +-- check maintenance daemon is started +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + datname | current_database | usename | extowner +--------------------------------------------------------------------- + regression | regression | postgres | postgres +(1 row) + +-- ensure no unexpected objects were created outside pg_catalog +SELECT pgio.type, pgio.identity +FROM pg_depend AS pgd, + pg_extension AS pge, + LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio +WHERE pgd.refclassid = 'pg_extension'::regclass AND + pgd.refobjid = pge.oid AND + pge.extname = 'citus' AND + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar') +ORDER BY 1, 2; + type | identity +--------------------------------------------------------------------- + view | public.citus_tables +(1 row) + +-- DROP EXTENSION pre-created by the regression suite +DROP EXTENSION citus; +\c +-- these tests switch between citus versions and call ddl's that require pg_dist_object to be created +SET citus.enable_object_propagation TO 'false'; +SET citus.enable_version_checks TO 'false'; +CREATE EXTENSION citus VERSION '8.0-1'; +ALTER EXTENSION citus UPDATE TO '8.0-2'; +ALTER EXTENSION citus UPDATE TO '8.0-3'; +ALTER EXTENSION citus UPDATE TO '8.0-4'; +ALTER EXTENSION citus UPDATE TO '8.0-5'; +ALTER EXTENSION citus UPDATE TO '8.0-6'; +ALTER EXTENSION citus UPDATE TO '8.0-7'; +ALTER EXTENSION citus UPDATE TO '8.0-8'; +ALTER EXTENSION citus UPDATE TO '8.0-9'; +ALTER EXTENSION citus UPDATE TO '8.0-10'; +ALTER EXTENSION citus UPDATE TO '8.0-11'; +ALTER EXTENSION citus UPDATE TO '8.0-12'; +ALTER EXTENSION citus UPDATE TO '8.0-13'; +ALTER EXTENSION citus UPDATE TO '8.1-1'; +ALTER EXTENSION citus UPDATE TO '8.2-1'; +ALTER EXTENSION citus UPDATE TO '8.2-2'; +ALTER EXTENSION citus UPDATE TO '8.2-3'; +ALTER EXTENSION citus UPDATE TO '8.2-4'; +ALTER EXTENSION citus UPDATE TO '8.3-1'; +ALTER EXTENSION citus UPDATE TO '9.0-1'; +ALTER EXTENSION citus UPDATE TO '9.0-2'; +ALTER EXTENSION citus UPDATE TO '9.1-1'; +ALTER EXTENSION citus UPDATE TO '9.2-1'; +ALTER EXTENSION citus UPDATE TO '9.2-2'; +-- Snapshot of state at 9.2-2 +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + | event trigger citus_cascade_to_partition + | function alter_role_if_exists(text,text) + | function any_value(anyelement) + | function any_value_agg(anyelement,anyelement) + | function array_cat_agg(anyarray) + | function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) + | function authinfo_valid(text) + | function broadcast_intermediate_result(text,text) + | function check_distributed_deadlocks() + | function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real) + | function citus_blocking_pids(integer) + | function citus_create_restore_point(text) + | function citus_dist_stat_activity() + | function citus_drop_trigger() + | function citus_executor_name(integer) + | function citus_extradata_container(internal) + | function citus_finish_pg_upgrade() + | function citus_internal.find_groupid_for_node(text,integer) + | function citus_internal.pg_dist_node_trigger_func() + | function citus_internal.pg_dist_rebalance_strategy_enterprise_check() + | function citus_internal.pg_dist_rebalance_strategy_trigger_func() + | function citus_internal.pg_dist_shard_placement_trigger_func() + | function citus_internal.refresh_isolation_tester_prepared_statement() + | function citus_internal.replace_isolation_tester_func() + | function citus_internal.restore_isolation_tester_func() + | function citus_isolation_test_session_is_blocked(integer,integer[]) + | function citus_json_concatenate(json,json) + | function citus_json_concatenate_final(json) + | function citus_jsonb_concatenate(jsonb,jsonb) + | function citus_jsonb_concatenate_final(jsonb) + | function citus_node_capacity_1(integer) + | function citus_prepare_pg_upgrade() + | function citus_query_stats() + | function citus_relation_size(regclass) + | function citus_server_id() + | function citus_set_default_rebalance_strategy(text) + | function citus_shard_allowed_on_node_true(bigint,integer) + | function citus_shard_cost_1(bigint) + | function citus_shard_cost_by_disk_size(bigint) + | function citus_stat_statements() + | function citus_stat_statements_reset() + | function citus_table_is_visible(oid) + | function citus_table_size(regclass) + | function citus_text_send_as_jsonb(text) + | function citus_total_relation_size(regclass) + | function citus_truncate_trigger() + | function citus_validate_rebalance_strategy_functions(regproc,regproc,regproc) + | function citus_version() + | function citus_worker_stat_activity() + | function column_name_to_column(regclass,text) + | function column_to_column_name(regclass,text) + | function coord_combine_agg(oid,cstring,anyelement) + | function coord_combine_agg_ffunc(internal,oid,cstring,anyelement) + | function coord_combine_agg_sfunc(internal,oid,cstring,anyelement) + | function create_distributed_function(regprocedure,text,text) + | function create_distributed_table(regclass,text,citus.distribution_type,text) + | function create_intermediate_result(text,text) + | function create_reference_table(regclass) + | function distributed_tables_colocated(regclass,regclass) + | function dump_global_wait_edges() + | function dump_local_wait_edges() + | function fetch_intermediate_results(text[],text,integer) + | function get_all_active_transactions() + | function get_colocated_shard_array(bigint) + | function get_colocated_table_array(regclass) + | function get_current_transaction_id() + | function get_global_active_transactions() + | function get_rebalance_progress() + | function get_rebalance_table_shards_plan(regclass,real,integer,bigint[],boolean,name) + | function get_shard_id_for_distribution_column(regclass,"any") + | function isolate_tenant_to_new_shard(regclass,"any",text) + | function json_cat_agg(json) + | function jsonb_cat_agg(jsonb) + | function lock_relation_if_exists(text,text) + | function lock_shard_metadata(integer,bigint[]) + | function lock_shard_resources(integer,bigint[]) + | function mark_tables_colocated(regclass,regclass[]) + | function master_activate_node(text,integer) + | function master_add_inactive_node(text,integer,integer,noderole,name) + | function master_add_node(text,integer,integer,noderole,name) + | function master_add_secondary_node(text,integer,text,integer,name) + | function master_append_table_to_shard(bigint,text,text,integer) + | function master_apply_delete_command(text) + | function master_conninfo_cache_invalidate() + | function master_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) + | function master_create_distributed_table(regclass,text,citus.distribution_type) + | function master_create_empty_shard(text) + | function master_create_worker_shards(text,integer,integer) + | function master_disable_node(text,integer) + | function master_dist_local_group_cache_invalidate() + | function master_dist_node_cache_invalidate() + | function master_dist_object_cache_invalidate() + | function master_dist_partition_cache_invalidate() + | function master_dist_placement_cache_invalidate() + | function master_dist_shard_cache_invalidate() + | function master_drain_node(text,integer,citus.shard_transfer_mode,name) + | function master_drop_all_shards(regclass,text,text) + | function master_drop_sequences(text[]) + | function master_get_active_worker_nodes() + | function master_get_new_placementid() + | function master_get_new_shardid() + | function master_get_table_ddl_events(text) + | function master_get_table_metadata(text) + | function master_modify_multiple_shards(text) + | function master_move_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) + | function master_remove_distributed_table_metadata_from_workers(regclass,text,text) + | function master_remove_node(text,integer) + | function master_remove_partition_metadata(regclass,text,text) + | function master_run_on_worker(text[],integer[],text[],boolean) + | function master_set_node_property(text,integer,text,boolean) + | function master_unmark_object_distributed(oid,oid,integer) + | function master_update_node(integer,text,integer,boolean,integer) + | function master_update_shard_statistics(bigint) + | function master_update_table_statistics(regclass) + | function poolinfo_valid(text) + | function read_intermediate_result(text,citus_copy_format) + | function read_intermediate_results(text[],citus_copy_format) + | function rebalance_table_shards(regclass,real,integer,bigint[],citus.shard_transfer_mode,boolean,name) + | function recover_prepared_transactions() + | function relation_is_a_known_shard(regclass) + | function replicate_table_shards(regclass,integer,integer,bigint[],citus.shard_transfer_mode) + | function role_exists(name) + | function run_command_on_colocated_placements(regclass,regclass,text,boolean) + | function run_command_on_placements(regclass,text,boolean) + | function run_command_on_shards(regclass,text,boolean) + | function run_command_on_workers(text,boolean) + | function shard_name(regclass,bigint) + | function start_metadata_sync_to_node(text,integer) + | function stop_metadata_sync_to_node(text,integer) + | function task_tracker_assign_task(bigint,integer,text) + | function task_tracker_cleanup_job(bigint) + | function task_tracker_conninfo_cache_invalidate() + | function task_tracker_task_status(bigint,integer) + | function upgrade_to_reference_table(regclass) + | function worker_append_table_to_shard(text,text,text,integer) + | function worker_apply_inter_shard_ddl_command(bigint,text,bigint,text,text) + | function worker_apply_sequence_command(text) + | function worker_apply_sequence_command(text,regtype) + | function worker_apply_shard_ddl_command(bigint,text) + | function worker_apply_shard_ddl_command(bigint,text,text) + | function worker_cleanup_job_schema_cache() + | function worker_create_or_replace_object(text) + | function worker_create_schema(bigint,text) + | function worker_create_truncate_trigger(regclass) + | function worker_drop_distributed_table(text) + | function worker_execute_sql_task(bigint,integer,text,boolean) + | function worker_fetch_foreign_file(text,text,bigint,text[],integer[]) + | function worker_fetch_partition_file(bigint,integer,integer,integer,text,integer) + | function worker_hash("any") + | function worker_hash_partition_table(bigint,integer,text,text,oid,anyarray) + | function worker_merge_files_and_run_query(bigint,integer,text,text) + | function worker_merge_files_into_table(bigint,integer,text[],text[]) + | function worker_partial_agg(oid,anyelement) + | function worker_partial_agg_ffunc(internal) + | function worker_partial_agg_sfunc(internal,oid,anyelement) + | function worker_partition_query_result(text,text,integer,citus.distribution_type,text[],text[],boolean) + | function worker_range_partition_table(bigint,integer,text,text,oid,anyarray) + | function worker_repartition_cleanup(bigint) + | schema citus + | schema citus_internal + | sequence pg_dist_colocationid_seq + | sequence pg_dist_groupid_seq + | sequence pg_dist_node_nodeid_seq + | sequence pg_dist_placement_placementid_seq + | sequence pg_dist_shardid_seq + | table citus.pg_dist_object + | table pg_dist_authinfo + | table pg_dist_colocation + | table pg_dist_local_group + | table pg_dist_node + | table pg_dist_node_metadata + | table pg_dist_partition + | table pg_dist_placement + | table pg_dist_poolinfo + | table pg_dist_rebalance_strategy + | table pg_dist_shard + | table pg_dist_transaction + | type citus.distribution_type + | type citus.shard_transfer_mode + | type citus_copy_format + | type noderole + | view citus_dist_stat_activity + | view citus_lock_waits + | view citus_shard_indexes_on_worker + | view citus_shards_on_worker + | view citus_stat_statements + | view citus_worker_stat_activity + | view pg_dist_shard_placement +(188 rows) + +-- Test downgrade to 9.2-2 from 9.2-4 +ALTER EXTENSION citus UPDATE TO '9.2-4'; +ALTER EXTENSION citus UPDATE TO '9.2-2'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +/* + * As we mistakenly bumped schema version to 9.3-1 in a bad release, we support + * updating citus schema from 9.3-1 to 9.2-4, but we do not support updates to 9.3-1. + * + * Hence the query below should fail. + */ +ALTER EXTENSION citus UPDATE TO '9.3-1'; +ERROR: extension "citus" has no update path from version "9.2-2" to version "9.3-1" +ALTER EXTENSION citus UPDATE TO '9.2-4'; +-- Snapshot of state at 9.2-4 +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Test downgrade to 9.2-4 from 9.3-2 +ALTER EXTENSION citus UPDATE TO '9.3-2'; +ALTER EXTENSION citus UPDATE TO '9.2-4'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 9.3-2 +ALTER EXTENSION citus UPDATE TO '9.3-2'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + | function citus_remote_connection_stats() + | function replicate_reference_tables() + | function truncate_local_data_after_distributing_table(regclass) + | function update_distributed_table_colocation(regclass,text) + | function worker_create_or_alter_role(text,text,text) +(5 rows) + +-- Test downgrade to 9.3-2 from 9.4-1 +ALTER EXTENSION citus UPDATE TO '9.4-1'; +ALTER EXTENSION citus UPDATE TO '9.3-2'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 9.4-1 +ALTER EXTENSION citus UPDATE TO '9.4-1'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + | function worker_last_saved_explain_analyze() + | function worker_save_query_explain_analyze(text,jsonb) +(2 rows) + +-- Test downgrade to 9.4-1 from 9.5-1 +ALTER EXTENSION citus UPDATE TO '9.5-1'; +BEGIN; + SELECT master_add_node('localhost', :master_port, groupId=>0); + master_add_node +--------------------------------------------------------------------- + 1 +(1 row) + + CREATE TABLE citus_local_table (a int); + SELECT create_citus_local_table('citus_local_table'); +NOTICE: create_citus_local_table is deprecated in favour of citus_add_local_table_to_metadata + create_citus_local_table +--------------------------------------------------------------------- + +(1 row) + + -- downgrade from 9.5-1 to 9.4-1 should fail as we have a citus local table + ALTER EXTENSION citus UPDATE TO '9.4-1'; +ERROR: citus local tables are introduced in Citus 9.5 +HINT: To downgrade Citus to an older version, you should first convert each citus local table to a postgres table by executing SELECT undistribute_table("%s") +CONTEXT: PL/pgSQL function inline_code_block line 11 at RAISE +ROLLBACK; +-- now we can downgrade as there is no citus local table +ALTER EXTENSION citus UPDATE TO '9.4-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 9.5-1 +ALTER EXTENSION citus UPDATE TO '9.5-1'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + function master_drop_sequences(text[]) | + function task_tracker_assign_task(bigint,integer,text) | + function task_tracker_cleanup_job(bigint) | + function task_tracker_conninfo_cache_invalidate() | + function task_tracker_task_status(bigint,integer) | + function worker_execute_sql_task(bigint,integer,text,boolean) | + function worker_merge_files_and_run_query(bigint,integer,text,text) | + | function create_citus_local_table(regclass) + | function undistribute_table(regclass) + | function worker_record_sequence_dependency(regclass,regclass,name) +(10 rows) + +-- Test downgrade to 9.5-1 from 10.0-1 +ALTER EXTENSION citus UPDATE TO '10.0-1'; +ALTER EXTENSION citus UPDATE TO '9.5-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 10.0-1 +ALTER EXTENSION citus UPDATE TO '10.0-1'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + function citus_total_relation_size(regclass) | + function create_citus_local_table(regclass) | + function mark_tables_colocated(regclass,regclass[]) | + function master_conninfo_cache_invalidate() | + function master_create_distributed_table(regclass,text,citus.distribution_type) | + function master_create_worker_shards(text,integer,integer) | + function master_dist_local_group_cache_invalidate() | + function master_dist_node_cache_invalidate() | + function master_dist_object_cache_invalidate() | + function master_dist_partition_cache_invalidate() | + function master_dist_placement_cache_invalidate() | + function master_dist_shard_cache_invalidate() | + function master_drop_all_shards(regclass,text,text) | + function master_modify_multiple_shards(text) | + function undistribute_table(regclass) | + function upgrade_to_reference_table(regclass) | + | access method columnar + | function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) + | function alter_columnar_table_set(regclass,integer,integer,name,integer) + | function alter_distributed_table(regclass,text,integer,text,boolean) + | function alter_old_partitions_set_access_method(regclass,timestamp with time zone,name) + | function alter_table_set_access_method(regclass,text) + | function citus_activate_node(text,integer) + | function citus_add_inactive_node(text,integer,integer,noderole,name) + | function citus_add_local_table_to_metadata(regclass,boolean) + | function citus_add_node(text,integer,integer,noderole,name) + | function citus_add_secondary_node(text,integer,text,integer,name) + | function citus_conninfo_cache_invalidate() + | function citus_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) + | function citus_disable_node(text,integer) + | function citus_dist_local_group_cache_invalidate() + | function citus_dist_node_cache_invalidate() + | function citus_dist_object_cache_invalidate() + | function citus_dist_partition_cache_invalidate() + | function citus_dist_placement_cache_invalidate() + | function citus_dist_shard_cache_invalidate() + | function citus_drain_node(text,integer,citus.shard_transfer_mode,name) + | function citus_drop_all_shards(regclass,text,text) + | function citus_internal.columnar_ensure_objects_exist() + | function citus_move_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) + | function citus_remove_node(text,integer) + | function citus_set_coordinator_host(text,integer,noderole,name) + | function citus_set_node_property(text,integer,text,boolean) + | function citus_shard_sizes() + | function citus_total_relation_size(regclass,boolean) + | function citus_unmark_object_distributed(oid,oid,integer) + | function citus_update_node(integer,text,integer,boolean,integer) + | function citus_update_shard_statistics(bigint) + | function citus_update_table_statistics(regclass) + | function columnar.columnar_handler(internal) + | function fix_pre_citus10_partitioned_table_constraint_names() + | function fix_pre_citus10_partitioned_table_constraint_names(regclass) + | function notify_constraint_dropped() + | function remove_local_tables_from_metadata() + | function time_partition_range(regclass) + | function undistribute_table(regclass,boolean) + | function worker_change_sequence_dependency(regclass,regclass,regclass) + | function worker_fix_pre_citus10_partitioned_table_constraint_names(regclass,bigint,text) + | schema columnar + | sequence columnar.storageid_seq + | table columnar.chunk + | table columnar.chunk_group + | table columnar.options + | table columnar.stripe + | view citus_shards + | view citus_tables + | view time_partitions +(67 rows) + +-- Test downgrade to 10.0-1 from 10.0-2 +ALTER EXTENSION citus UPDATE TO '10.0-2'; +ALTER EXTENSION citus UPDATE TO '10.0-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 10.0-2 +ALTER EXTENSION citus UPDATE TO '10.0-2'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Test downgrade to 10.0-2 from 10.0-3 +ALTER EXTENSION citus UPDATE TO '10.0-3'; +ALTER EXTENSION citus UPDATE TO '10.0-2'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 10.0-3 +ALTER EXTENSION citus UPDATE TO '10.0-3'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + | function citus_get_active_worker_nodes() +(1 row) + +-- Test downgrade to 10.0-3 from 10.1-1 +ALTER EXTENSION citus UPDATE TO '10.1-1'; +ALTER EXTENSION citus UPDATE TO '10.0-3'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 10.1-1 +ALTER EXTENSION citus UPDATE TO '10.1-1'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- + function create_distributed_table(regclass,text,citus.distribution_type,text) | + | function create_distributed_table(regclass,text,citus.distribution_type,text,integer) +(2 rows) + +DROP TABLE prev_objects, extension_diff; +-- show running version +SHOW citus.version; + citus.version +--------------------------------------------------------------------- + 10.1devel +(1 row) + +-- ensure no unexpected objects were created outside pg_catalog +SELECT pgio.type, pgio.identity +FROM pg_depend AS pgd, + pg_extension AS pge, + LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio +WHERE pgd.refclassid = 'pg_extension'::regclass AND + pgd.refobjid = pge.oid AND + pge.extname = 'citus' AND + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar') +ORDER BY 1, 2; + type | identity +--------------------------------------------------------------------- + view | public.citus_tables +(1 row) + +-- see incompatible version errors out +RESET citus.enable_version_checks; +DROP EXTENSION citus; +CREATE EXTENSION citus VERSION '8.0-1'; +ERROR: specified version incompatible with loaded Citus library +DETAIL: Loaded library requires 10.1, but 8.0-1 was specified. +HINT: If a newer library is present, restart the database and try the command again. +-- Test non-distributed queries work even in version mismatch +SET citus.enable_version_checks TO 'false'; +CREATE EXTENSION citus VERSION '8.1-1'; +SET citus.enable_version_checks TO 'true'; +-- Test CREATE TABLE +CREATE TABLE version_mismatch_table(column1 int); +-- Test COPY +\copy version_mismatch_table FROM STDIN; +-- Test INSERT +INSERT INTO version_mismatch_table(column1) VALUES(5); +-- Test SELECT +SELECT * FROM version_mismatch_table ORDER BY column1; + column1 +--------------------------------------------------------------------- + 0 + 1 + 2 + 3 + 4 + 5 +(6 rows) + +-- Test SELECT from pg_catalog +SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" +FROM pg_catalog.pg_database d +ORDER BY 1; + Name | Owner | Access privileges +--------------------------------------------------------------------- + postgres | postgres | + regression | postgres | + template0 | postgres | =c/postgres + + | | postgres=CTc/postgres + template1 | postgres | =c/postgres + + | | postgres=CTc/postgres +(4 rows) + +-- We should not distribute table in version mistmatch +SELECT create_distributed_table('version_mismatch_table', 'column1'); +ERROR: loaded Citus library version differs from installed extension version +DETAIL: Loaded library requires 10.1, but the installed extension version is 8.1-1. +HINT: Run ALTER EXTENSION citus UPDATE and try again. +-- This function will cause fail in next ALTER EXTENSION +CREATE OR REPLACE FUNCTION pg_catalog.relation_is_a_known_shard(regclass) +RETURNS void LANGUAGE plpgsql +AS $function$ +BEGIN +END; +$function$; +ERROR: cannot change return type of existing function +HINT: Use DROP FUNCTION relation_is_a_known_shard(regclass) first. +SET citus.enable_version_checks TO 'false'; +-- This will fail because of previous function declaration +ALTER EXTENSION citus UPDATE TO '8.1-1'; +NOTICE: version "8.1-1" of extension "citus" is already installed +-- We can DROP problematic function and continue ALTER EXTENSION even when version checks are on +SET citus.enable_version_checks TO 'true'; +DROP FUNCTION pg_catalog.relation_is_a_known_shard(regclass); +ERROR: cannot drop function relation_is_a_known_shard(regclass) because extension citus requires it +HINT: You can drop extension citus instead. +SET citus.enable_version_checks TO 'false'; +ALTER EXTENSION citus UPDATE TO '8.1-1'; +NOTICE: version "8.1-1" of extension "citus" is already installed +-- Test updating to the latest version without specifying the version number +ALTER EXTENSION citus UPDATE; +-- re-create in newest version +DROP EXTENSION citus; +\c +CREATE EXTENSION citus; +-- test cache invalidation in workers +\c - - - :worker_1_port +DROP EXTENSION citus; +SET citus.enable_version_checks TO 'false'; +CREATE EXTENSION citus VERSION '8.0-1'; +SET citus.enable_version_checks TO 'true'; +-- during ALTER EXTENSION, we should invalidate the cache +ALTER EXTENSION citus UPDATE; +-- if cache is invalidated succesfull, this \d should work without any problem +\d + List of relations + Schema | Name | Type | Owner +--------------------------------------------------------------------- + public | citus_tables | view | postgres +(1 row) + +\c - - - :master_port +-- test https://github.com/citusdata/citus/issues/3409 +CREATE USER testuser2 SUPERUSER; +NOTICE: not propagating CREATE ROLE/USER commands to worker nodes +HINT: Connect to worker nodes directly to manually create all necessary users and roles. +SET ROLE testuser2; +DROP EXTENSION Citus; +-- Loop until we see there's no maintenance daemon running +DO $$begin + for i in 0 .. 100 loop + if i = 100 then raise 'Waited too long'; end if; + PERFORM pg_stat_clear_snapshot(); + perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon'; + if not found then exit; end if; + perform pg_sleep(0.1); + end loop; +end$$; +SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + datid | datname | usename +--------------------------------------------------------------------- +(0 rows) + +CREATE EXTENSION Citus; +-- Loop until we there's a maintenance daemon running +DO $$begin + for i in 0 .. 100 loop + if i = 100 then raise 'Waited too long'; end if; + PERFORM pg_stat_clear_snapshot(); + perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon'; + if found then exit; end if; + perform pg_sleep(0.1); + end loop; +end$$; +SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + datid | datname | usename +--------------------------------------------------------------------- + 16384 | regression | testuser2 +(1 row) + +RESET ROLE; +-- check that maintenance daemon gets (re-)started for the right user +DROP EXTENSION citus; +CREATE USER testuser SUPERUSER; +SET ROLE testuser; +CREATE EXTENSION citus; +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + datname | current_database | usename | extowner +--------------------------------------------------------------------- + regression | regression | testuser | testuser +(1 row) + +-- and recreate as the right owner +RESET ROLE; +DROP EXTENSION citus; +CREATE EXTENSION citus; +-- Check that maintenance daemon can also be started in another database +CREATE DATABASE another; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +\c another +CREATE EXTENSION citus; +CREATE SCHEMA test; +:create_function_test_maintenance_worker +-- see that the daemon started +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + datname | current_database | usename | extowner +--------------------------------------------------------------------- + another | another | postgres | postgres +(1 row) + +-- Test that database with active worker can be dropped. +\c regression +CREATE SCHEMA test_daemon; +-- we create a similar function on the regression database +-- note that this function checks for the existence of the daemon +-- when not found, returns true else tries for 5 times and +-- returns false +CREATE OR REPLACE FUNCTION test_daemon.maintenance_daemon_died(p_dbname text) + RETURNS boolean + LANGUAGE plpgsql +AS $$ +DECLARE + activity record; +BEGIN + PERFORM pg_stat_clear_snapshot(); + SELECT * INTO activity FROM pg_stat_activity + WHERE application_name = 'Citus Maintenance Daemon' AND datname = p_dbname; + IF activity.pid IS NULL THEN + RETURN true; + ELSE + RETURN false; + END IF; +END; +$$; +-- drop the database and see that the daemon is dead +DROP DATABASE another; +SELECT + * +FROM + test_daemon.maintenance_daemon_died('another'); + maintenance_daemon_died +--------------------------------------------------------------------- + t +(1 row) + +-- we don't need the schema and the function anymore +DROP SCHEMA test_daemon CASCADE; +NOTICE: drop cascades to function test_daemon.maintenance_daemon_died(text) +-- verify citus does not crash while creating a table when run against an older worker +-- create_distributed_table piggybacks multiple commands into single one, if one worker +-- did not have the required UDF it should fail instead of crash. +-- create a test database, configure citus with single node +CREATE DATABASE another; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +\c - - - :worker_1_port +CREATE DATABASE another; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +\c - - - :master_port +\c another +CREATE EXTENSION citus; +SET citus.enable_object_propagation TO off; -- prevent distributed transactions during add node +SELECT FROM master_add_node('localhost', :worker_1_port); +WARNING: citus.enable_object_propagation is off, not creating distributed objects on worker +DETAIL: distributed objects are only kept in sync when citus.enable_object_propagation is set to on. Newly activated nodes will not get these objects created +-- +(1 row) + +\c - - - :worker_1_port +CREATE EXTENSION citus; +ALTER FUNCTION assign_distributed_transaction_id(initiator_node_identifier integer, transaction_number bigint, transaction_stamp timestamp with time zone) +RENAME TO dummy_assign_function; +\c - - - :master_port +SET citus.shard_replication_factor to 1; +-- create_distributed_table command should fail +CREATE TABLE t1(a int, b int); +SET client_min_messages TO ERROR; +DO $$ +BEGIN + BEGIN + SELECT create_distributed_table('t1', 'a'); + EXCEPTION WHEN OTHERS THEN + RAISE 'create distributed table failed'; + END; +END; +$$; +ERROR: create distributed table failed +CONTEXT: PL/pgSQL function inline_code_block line 6 at RAISE +\c regression +\c - - - :master_port +DROP DATABASE another; +\c - - - :worker_1_port +DROP DATABASE another; +\c - - - :master_port +-- only the regression database should have a maintenance daemon +SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- recreate the extension immediately after the maintenancae daemon errors +SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + pg_cancel_backend +--------------------------------------------------------------------- + t +(1 row) + +DROP EXTENSION citus; +CREATE EXTENSION citus; +-- wait for maintenance daemon restart +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + datname | current_database | usename | extowner +--------------------------------------------------------------------- + regression | regression | postgres | postgres +(1 row) + +-- confirm that there is only one maintenance daemon +SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- kill the maintenance daemon +SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + pg_cancel_backend +--------------------------------------------------------------------- + t +(1 row) + +-- reconnect +\c - - - :master_port +-- run something that goes through planner hook and therefore kicks of maintenance daemon +SELECT 1; + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- wait for maintenance daemon restart +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + datname | current_database | usename | extowner +--------------------------------------------------------------------- + regression | regression | postgres | postgres +(1 row) + +-- confirm that there is only one maintenance daemon +SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + count +--------------------------------------------------------------------- + 1 +(1 row) + +DROP TABLE version_mismatch_table; diff --git a/python/tests/projects/citus/src/test/regress/sql/multi_extension.sql b/python/tests/projects/citus/src/test/regress/sql/multi_extension.sql new file mode 100644 index 00000000..20e0e1df --- /dev/null +++ b/python/tests/projects/citus/src/test/regress/sql/multi_extension.sql @@ -0,0 +1,498 @@ +-- +-- MULTI_EXTENSION +-- +-- Tests around extension creation / upgrades +-- +-- It'd be nice to script generation of this file, but alas, that's +-- not done yet. + +-- differentiate the output file for pg11 and versions above, with regards to objects +-- created per citus version depending on the postgres version. Upgrade tests verify the +-- objects are added in citus_finish_pg_upgrade() +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS version_above_eleven; + +SET citus.next_shard_id TO 580000; + +SELECT $definition$ +CREATE OR REPLACE FUNCTION test.maintenance_worker() + RETURNS pg_stat_activity + LANGUAGE plpgsql +AS $$ +DECLARE + activity record; +BEGIN + DO 'BEGIN END'; -- Force maintenance daemon to start + -- we don't want to wait forever; loop will exit after 20 seconds + FOR i IN 1 .. 200 LOOP + PERFORM pg_stat_clear_snapshot(); + SELECT * INTO activity FROM pg_stat_activity + WHERE application_name = 'Citus Maintenance Daemon' AND datname = current_database(); + IF activity.pid IS NOT NULL THEN + RETURN activity; + ELSE + PERFORM pg_sleep(0.1); + END IF ; + END LOOP; + -- fail if we reach the end of this loop + raise 'Waited too long for maintenance daemon to start'; +END; +$$; +$definition$ create_function_test_maintenance_worker +\gset + +CREATE TABLE prev_objects(description text); +CREATE TABLE extension_diff(previous_object text COLLATE "C", + current_object text COLLATE "C"); + +CREATE FUNCTION print_extension_changes() +RETURNS TABLE(previous_object text, current_object text) +AS $func$ +BEGIN + TRUNCATE TABLE extension_diff; + + CREATE TABLE current_objects AS + SELECT pg_catalog.pg_describe_object(classid, objid, 0) AS description + FROM pg_catalog.pg_depend, pg_catalog.pg_extension e + WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass + AND refobjid = e.oid + AND deptype = 'e' + AND e.extname='citus'; + + INSERT INTO extension_diff + SELECT p.description previous_object, c.description current_object + FROM current_objects c FULL JOIN prev_objects p + ON p.description = c.description + WHERE p.description is null OR c.description is null; + + DROP TABLE prev_objects; + ALTER TABLE current_objects RENAME TO prev_objects; + + RETURN QUERY SELECT * FROM extension_diff ORDER BY 1, 2; +END +$func$ LANGUAGE plpgsql; + +CREATE SCHEMA test; +:create_function_test_maintenance_worker + +-- check maintenance daemon is started +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + +-- ensure no unexpected objects were created outside pg_catalog +SELECT pgio.type, pgio.identity +FROM pg_depend AS pgd, + pg_extension AS pge, + LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio +WHERE pgd.refclassid = 'pg_extension'::regclass AND + pgd.refobjid = pge.oid AND + pge.extname = 'citus' AND + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar') +ORDER BY 1, 2; + + +-- DROP EXTENSION pre-created by the regression suite +DROP EXTENSION citus; +\c + +-- these tests switch between citus versions and call ddl's that require pg_dist_object to be created +SET citus.enable_object_propagation TO 'false'; + +SET citus.enable_version_checks TO 'false'; + +CREATE EXTENSION citus VERSION '8.0-1'; +ALTER EXTENSION citus UPDATE TO '8.0-2'; +ALTER EXTENSION citus UPDATE TO '8.0-3'; +ALTER EXTENSION citus UPDATE TO '8.0-4'; +ALTER EXTENSION citus UPDATE TO '8.0-5'; +ALTER EXTENSION citus UPDATE TO '8.0-6'; +ALTER EXTENSION citus UPDATE TO '8.0-7'; +ALTER EXTENSION citus UPDATE TO '8.0-8'; +ALTER EXTENSION citus UPDATE TO '8.0-9'; +ALTER EXTENSION citus UPDATE TO '8.0-10'; +ALTER EXTENSION citus UPDATE TO '8.0-11'; +ALTER EXTENSION citus UPDATE TO '8.0-12'; +ALTER EXTENSION citus UPDATE TO '8.0-13'; +ALTER EXTENSION citus UPDATE TO '8.1-1'; +ALTER EXTENSION citus UPDATE TO '8.2-1'; +ALTER EXTENSION citus UPDATE TO '8.2-2'; +ALTER EXTENSION citus UPDATE TO '8.2-3'; +ALTER EXTENSION citus UPDATE TO '8.2-4'; +ALTER EXTENSION citus UPDATE TO '8.3-1'; +ALTER EXTENSION citus UPDATE TO '9.0-1'; +ALTER EXTENSION citus UPDATE TO '9.0-2'; +ALTER EXTENSION citus UPDATE TO '9.1-1'; +ALTER EXTENSION citus UPDATE TO '9.2-1'; +ALTER EXTENSION citus UPDATE TO '9.2-2'; +-- Snapshot of state at 9.2-2 +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 9.2-2 from 9.2-4 +ALTER EXTENSION citus UPDATE TO '9.2-4'; +ALTER EXTENSION citus UPDATE TO '9.2-2'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +/* + * As we mistakenly bumped schema version to 9.3-1 in a bad release, we support + * updating citus schema from 9.3-1 to 9.2-4, but we do not support updates to 9.3-1. + * + * Hence the query below should fail. + */ +ALTER EXTENSION citus UPDATE TO '9.3-1'; + +ALTER EXTENSION citus UPDATE TO '9.2-4'; +-- Snapshot of state at 9.2-4 +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 9.2-4 from 9.3-2 +ALTER EXTENSION citus UPDATE TO '9.3-2'; +ALTER EXTENSION citus UPDATE TO '9.2-4'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at 9.3-2 +ALTER EXTENSION citus UPDATE TO '9.3-2'; +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 9.3-2 from 9.4-1 +ALTER EXTENSION citus UPDATE TO '9.4-1'; +ALTER EXTENSION citus UPDATE TO '9.3-2'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at 9.4-1 +ALTER EXTENSION citus UPDATE TO '9.4-1'; +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 9.4-1 from 9.5-1 +ALTER EXTENSION citus UPDATE TO '9.5-1'; + +BEGIN; + SELECT master_add_node('localhost', :master_port, groupId=>0); + CREATE TABLE citus_local_table (a int); + SELECT create_citus_local_table('citus_local_table'); + + -- downgrade from 9.5-1 to 9.4-1 should fail as we have a citus local table + ALTER EXTENSION citus UPDATE TO '9.4-1'; +ROLLBACK; + +-- now we can downgrade as there is no citus local table +ALTER EXTENSION citus UPDATE TO '9.4-1'; + +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at 9.5-1 +ALTER EXTENSION citus UPDATE TO '9.5-1'; +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 9.5-1 from 10.0-1 +ALTER EXTENSION citus UPDATE TO '10.0-1'; +ALTER EXTENSION citus UPDATE TO '9.5-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at 10.0-1 +ALTER EXTENSION citus UPDATE TO '10.0-1'; +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 10.0-1 from 10.0-2 +ALTER EXTENSION citus UPDATE TO '10.0-2'; +ALTER EXTENSION citus UPDATE TO '10.0-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at 10.0-2 +ALTER EXTENSION citus UPDATE TO '10.0-2'; +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 10.0-2 from 10.0-3 +ALTER EXTENSION citus UPDATE TO '10.0-3'; +ALTER EXTENSION citus UPDATE TO '10.0-2'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at 10.0-3 +ALTER EXTENSION citus UPDATE TO '10.0-3'; +SELECT * FROM print_extension_changes(); + +-- Test downgrade to 10.0-3 from 10.1-1 +ALTER EXTENSION citus UPDATE TO '10.1-1'; +ALTER EXTENSION citus UPDATE TO '10.0-3'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at 10.1-1 +ALTER EXTENSION citus UPDATE TO '10.1-1'; +SELECT * FROM print_extension_changes(); + +DROP TABLE prev_objects, extension_diff; + +-- show running version +SHOW citus.version; + +-- ensure no unexpected objects were created outside pg_catalog +SELECT pgio.type, pgio.identity +FROM pg_depend AS pgd, + pg_extension AS pge, + LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio +WHERE pgd.refclassid = 'pg_extension'::regclass AND + pgd.refobjid = pge.oid AND + pge.extname = 'citus' AND + pgio.schema NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar') +ORDER BY 1, 2; + +-- see incompatible version errors out +RESET citus.enable_version_checks; +DROP EXTENSION citus; +CREATE EXTENSION citus VERSION '8.0-1'; + +-- Test non-distributed queries work even in version mismatch +SET citus.enable_version_checks TO 'false'; +CREATE EXTENSION citus VERSION '8.1-1'; +SET citus.enable_version_checks TO 'true'; + +-- Test CREATE TABLE +CREATE TABLE version_mismatch_table(column1 int); + +-- Test COPY +\copy version_mismatch_table FROM STDIN; +0 +1 +2 +3 +4 +\. + +-- Test INSERT +INSERT INTO version_mismatch_table(column1) VALUES(5); + +-- Test SELECT +SELECT * FROM version_mismatch_table ORDER BY column1; + +-- Test SELECT from pg_catalog +SELECT d.datname as "Name", + pg_catalog.pg_get_userbyid(d.datdba) as "Owner", + pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges" +FROM pg_catalog.pg_database d +ORDER BY 1; + +-- We should not distribute table in version mistmatch +SELECT create_distributed_table('version_mismatch_table', 'column1'); + +-- This function will cause fail in next ALTER EXTENSION +CREATE OR REPLACE FUNCTION pg_catalog.relation_is_a_known_shard(regclass) +RETURNS void LANGUAGE plpgsql +AS $function$ +BEGIN +END; +$function$; + +SET citus.enable_version_checks TO 'false'; +-- This will fail because of previous function declaration +ALTER EXTENSION citus UPDATE TO '8.1-1'; + +-- We can DROP problematic function and continue ALTER EXTENSION even when version checks are on +SET citus.enable_version_checks TO 'true'; +DROP FUNCTION pg_catalog.relation_is_a_known_shard(regclass); + +SET citus.enable_version_checks TO 'false'; +ALTER EXTENSION citus UPDATE TO '8.1-1'; + +-- Test updating to the latest version without specifying the version number +ALTER EXTENSION citus UPDATE; + +-- re-create in newest version +DROP EXTENSION citus; +\c +CREATE EXTENSION citus; + +-- test cache invalidation in workers +\c - - - :worker_1_port + +DROP EXTENSION citus; +SET citus.enable_version_checks TO 'false'; +CREATE EXTENSION citus VERSION '8.0-1'; +SET citus.enable_version_checks TO 'true'; +-- during ALTER EXTENSION, we should invalidate the cache +ALTER EXTENSION citus UPDATE; + +-- if cache is invalidated succesfull, this \d should work without any problem +\d + +\c - - - :master_port + +-- test https://github.com/citusdata/citus/issues/3409 +CREATE USER testuser2 SUPERUSER; +SET ROLE testuser2; +DROP EXTENSION Citus; +-- Loop until we see there's no maintenance daemon running +DO $$begin + for i in 0 .. 100 loop + if i = 100 then raise 'Waited too long'; end if; + PERFORM pg_stat_clear_snapshot(); + perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon'; + if not found then exit; end if; + perform pg_sleep(0.1); + end loop; +end$$; +SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; +CREATE EXTENSION Citus; +-- Loop until we there's a maintenance daemon running +DO $$begin + for i in 0 .. 100 loop + if i = 100 then raise 'Waited too long'; end if; + PERFORM pg_stat_clear_snapshot(); + perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon'; + if found then exit; end if; + perform pg_sleep(0.1); + end loop; +end$$; +SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; +RESET ROLE; + +-- check that maintenance daemon gets (re-)started for the right user +DROP EXTENSION citus; +CREATE USER testuser SUPERUSER; +SET ROLE testuser; +CREATE EXTENSION citus; + +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + +-- and recreate as the right owner +RESET ROLE; +DROP EXTENSION citus; +CREATE EXTENSION citus; + + +-- Check that maintenance daemon can also be started in another database +CREATE DATABASE another; +\c another +CREATE EXTENSION citus; + +CREATE SCHEMA test; +:create_function_test_maintenance_worker + +-- see that the daemon started +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + +-- Test that database with active worker can be dropped. +\c regression + +CREATE SCHEMA test_daemon; + +-- we create a similar function on the regression database +-- note that this function checks for the existence of the daemon +-- when not found, returns true else tries for 5 times and +-- returns false +CREATE OR REPLACE FUNCTION test_daemon.maintenance_daemon_died(p_dbname text) + RETURNS boolean + LANGUAGE plpgsql +AS $$ +DECLARE + activity record; +BEGIN + PERFORM pg_stat_clear_snapshot(); + SELECT * INTO activity FROM pg_stat_activity + WHERE application_name = 'Citus Maintenance Daemon' AND datname = p_dbname; + IF activity.pid IS NULL THEN + RETURN true; + ELSE + RETURN false; + END IF; +END; +$$; + +-- drop the database and see that the daemon is dead +DROP DATABASE another; +SELECT + * +FROM + test_daemon.maintenance_daemon_died('another'); + +-- we don't need the schema and the function anymore +DROP SCHEMA test_daemon CASCADE; + + +-- verify citus does not crash while creating a table when run against an older worker +-- create_distributed_table piggybacks multiple commands into single one, if one worker +-- did not have the required UDF it should fail instead of crash. + +-- create a test database, configure citus with single node +CREATE DATABASE another; +\c - - - :worker_1_port +CREATE DATABASE another; +\c - - - :master_port + +\c another +CREATE EXTENSION citus; +SET citus.enable_object_propagation TO off; -- prevent distributed transactions during add node +SELECT FROM master_add_node('localhost', :worker_1_port); + +\c - - - :worker_1_port +CREATE EXTENSION citus; +ALTER FUNCTION assign_distributed_transaction_id(initiator_node_identifier integer, transaction_number bigint, transaction_stamp timestamp with time zone) +RENAME TO dummy_assign_function; + +\c - - - :master_port +SET citus.shard_replication_factor to 1; +-- create_distributed_table command should fail +CREATE TABLE t1(a int, b int); +SET client_min_messages TO ERROR; +DO $$ +BEGIN + BEGIN + SELECT create_distributed_table('t1', 'a'); + EXCEPTION WHEN OTHERS THEN + RAISE 'create distributed table failed'; + END; +END; +$$; + +\c regression +\c - - - :master_port +DROP DATABASE another; + +\c - - - :worker_1_port +DROP DATABASE another; + +\c - - - :master_port +-- only the regression database should have a maintenance daemon +SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + +-- recreate the extension immediately after the maintenancae daemon errors +SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; +DROP EXTENSION citus; +CREATE EXTENSION citus; + +-- wait for maintenance daemon restart +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + +-- confirm that there is only one maintenance daemon +SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + +-- kill the maintenance daemon +SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + +-- reconnect +\c - - - :master_port +-- run something that goes through planner hook and therefore kicks of maintenance daemon +SELECT 1; + +-- wait for maintenance daemon restart +SELECT datname, current_database(), + usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus') +FROM test.maintenance_worker(); + +-- confirm that there is only one maintenance daemon +SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon'; + +DROP TABLE version_mismatch_table; diff --git a/python/tests/test_common_tool_methods.py b/python/tests/test_common_tool_methods.py new file mode 100644 index 00000000..aacd2081 --- /dev/null +++ b/python/tests/test_common_tool_methods.py @@ -0,0 +1,81 @@ +import os +import unittest +from shutil import copyfile + +import pathlib2 +from github import Github + +from ..common_tool_methods import * + +GITHUB_TOKEN = os.getenv("GH_TOKEN") +TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() + + +class CommonToolMethodsTestCases(unittest.TestCase): + def test_get_version_number(self): + self.assertEqual(get_version_number("10.0.3", True, 1), "10.0.3-1") + + def test_get_version_number_with_project_name(self): + self.assertEqual(get_version_number_with_project_name("citus", "10.0.3", True, 1), "10.0.3.citus-1") + + def test_find_nth_overlapping(self): + self.assertEqual(find_nth_overlapping("foofoo foofoo", "foofoo", 2), 7) + + def test_find_nth_overlapping_line_by_regex(self): + # Two match case + self.assertEqual(find_nth_overlapping_line_by_regex("citusx\n citusx\ncitusx", "^citusx$", 2), 2) + # No match case + self.assertEqual(find_nth_overlapping_line_by_regex("citusx\n citusx\ncitusx", "^citusy$", 2), -1) + + def test_is_major_release(self): + self.assertEqual(True, is_major_release("10.0.0")) + self.assertEqual(False, is_major_release("10.0.1")) + + def test_str_array_to_str(self): + self.assertEqual("1\n2\n3\n4\n", str_array_to_str(["1", "2", "3", "4"])) + + def test_run(self): + result = run("echo 'Run' method is performing fine ") + self.assertEqual(0, result.returncode) + + def test_remove_parentheses_from_string(self): + self.assertEqual("out of parentheses ", + remove_string_inside_parentheses("out of parentheses (inside parentheses)")) + + def test_get_version_details(self): + self.assertEqual({"major": "10", "minor": "0", "patch": "1"}, get_version_details("10.0.1")) + + def test_replace_line_in_file(self): + replace_str = "Summary: Replace Test" + copy_file_path = f"{TEST_BASE_PATH}/files/citus_copy.spec" + copyfile(f"{TEST_BASE_PATH}/files/citus.spec", copy_file_path) + replace_line_in_file(copy_file_path, r"^Summary: *", replace_str) + try: + with open(copy_file_path, "r") as reader: + content = reader.read() + lines = content.splitlines() + self.assertEqual(lines[5], replace_str) + finally: + os.remove(copy_file_path) + + def test_getprs(self): + # created at is not seen on Github. Should be checked on API result + g = Github(GITHUB_TOKEN) + repository = g.get_repo(f"citusdata/citus") + prs = get_prs(repository, datetime.strptime('2021.02.26', '%Y.%m.%d'), "master", + datetime.strptime('2021.03.02', '%Y.%m.%d')) + self.assertEqual(5, len(prs)) + self.assertEqual(4760, prs[0].number) + + def test_getprs_with_backlog_label(self): + g = Github(GITHUB_TOKEN) + repository = g.get_repo(f"citusdata/citus") + prs = get_prs(repository, datetime.strptime('2021.02.20', '%Y.%m.%d'), "master", + datetime.strptime('2021.02.27', '%Y.%m.%d')) + prs_backlog = get_pr_issues_by_label(prs, "backport") + self.assertEqual(1, len(prs_backlog)) + self.assertEqual(4746, prs_backlog[0].number) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py new file mode 100644 index 00000000..3fd26b83 --- /dev/null +++ b/python/tests/test_prepare_release.py @@ -0,0 +1,28 @@ +import shutil +import unittest + +import pathlib2 + +from ..prepare_release import * + +github_token = os.getenv("GH_TOKEN") + +TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() +CITUS_PROJECT_TEST_PATH = f"{TEST_BASE_PATH}/projects/citus" +CITUS_PROJECT_TEST_PATH_COPY = f"{TEST_BASE_PATH}/projects/citus_copy" + + +class PrepareReleaseTestCases(unittest.TestCase): + def test_major_release(self): + shutil.copytree(CITUS_PROJECT_TEST_PATH, CITUS_PROJECT_TEST_PATH_COPY) + try: + update_release(github_token=github_token, project_name="citus", project_version="10.1.0", + main_branch="master", + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=CITUS_PROJECT_TEST_PATH_COPY) + # current branch is release-$version number + # check for configure in + # check for + finally: + # shutil.rmtree(CITUS_PROJECT_TEST_PATH_COPY) + run(f"rm -r {CITUS_PROJECT_TEST_PATH_COPY}") diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index 8541ed93..da99518e 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -1,6 +1,5 @@ import os import unittest -from datetime import datetime from shutil import copyfile from .test_utils import are_strings_equal @@ -20,15 +19,7 @@ class PackagePropertiesTestCases(unittest.TestCase): - def test_get_version_number(self): - self.assertEqual(get_version_number("10.0.3", True, 1), "10.0.3-1") - print(os.getenv("test")) - def test_get_version_number_with_project_name(self): - self.assertEqual(get_version_number_with_project_name("citus", "10.0.3", True, 1), "10.0.3.citus-1") - - def test_find_nth_overlapping(self): - self.assertEqual(find_nth_overlapping("foofoo foofoo", "foofoo", 2), 7) def test_get_changelog_for_tag(self): changelog = get_changelog_for_tag(GITHUB_TOKEN, "citus", "v10.0.3") diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 2a48b2fe..d4915a2b 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -1,91 +1,13 @@ import argparse -import re from datetime import date, datetime - import pathlib2 -import string_utils -from github import Github -from jinja2 import Environment, FileSystemLoader -from parameters_validation import no_whitespaces, non_blank, non_empty, non_negative, validate_parameters, \ - parameter_validation - -BASE_PATH = pathlib2.Path(__file__).parent.absolute() - - -@parameter_validation -def is_version(version: str): - if version is None or not version: - raise ValueError("version should be non-empty and should not be None") - if not re.match(r"\d+\.\d+\.\d+$", version): - raise ValueError( - "version should include 3 levels of versions consists of numbers separated with dots. e.g: 10.0.1") - - -@parameter_validation -def is_tag(tag: str): - if tag is None or not tag: - raise ValueError("tag should be non-empty and should not be None") - if not re.match(r"v\d+\.\d+\.\d+$", tag): - raise ValueError( - "tag should start with v and should include 3 levels of versions consists of numbers " + - "separated with dots. e.g: v10.0.1") - - -@parameter_validation -def is_email(email: str): - if not string_utils.is_email(email): - raise ValueError("Parameter is not in email format") - - -@parameter_validation -def is_project_changelog_header(header: str): - if header is None or not header: - raise ValueError("header should be non-empty and should not be None") - if not re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", header): - raise ValueError( - f"changelog header is in invalid format. Actual:{header} Expected: ### citus v8.3.3 (March 23, 2021) ### ") - - -def get_spec_file_name(project_name: str) -> str: - return f"{project_name}.spec" - - -def get_version_number(version: str, fancy: bool, fancy_release_count: int) -> str: - fancy_suffix = f"-{fancy_release_count}" if fancy else "" - return f"{version}{fancy_suffix}" - - -def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, fancy_release_count: int) -> str: - fancy_suffix = f"-{fancy_release_count}" if fancy else "" - return f"{version}.{project_name}{fancy_suffix}" - - -def get_template_environment(template_dir: str) -> Environment: - file_loader = FileSystemLoader(template_dir) - env = Environment(loader=file_loader) - return env - -def find_nth_overlapping(subject_string, search_string, n) -> int: - start = subject_string.find(search_string) - - while start >= 0 and n > 1: - start = subject_string.find(search_string, start + 1) - n -= 1 - return start +from github import Github +from parameters_validation import no_whitespaces, non_blank, non_empty, non_negative, validate_parameters +from .common_tool_methods import * -def find_nth_overlapping_line_by_regex(subject_string, regex_pattern, n) -> int: - lines = subject_string.splitlines() - counter = 0 - index = -1 - for i in range(len(lines)): - if re.match(regex_pattern, lines[i]): - counter = counter + 1 - if counter == n: - index = i - break - return index +BASE_PATH = pathlib2.Path(__file__).parent.absolute() def get_last_changelog_content(all_changelog_content: str) -> str: @@ -109,8 +31,13 @@ def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: return changelogs -def remove_parentheses_from_string(param: str) -> str: - return re.sub(r"[(\[].*?[)\]]", "", param) +@parameter_validation +def is_project_changelog_header(header: str): + if header is None or not header: + raise ValueError("header should be non-empty and should not be None") + if not re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", header): + raise ValueError( + f"changelog header is in invalid format. Actual:{header} Expected: ### citus v8.3.3 (March 23, 2021) ### ") def get_changelog_for_tag(github_token: str, project_name: str, tag_name: str) -> str: @@ -128,7 +55,7 @@ def get_changelog_for_tag(github_token: str, project_name: str, tag_name: str) - def get_debian_changelog_header(changelog_header: is_project_changelog_header(str), fancy: bool, fancy_version_number: int) -> str: hash_removed_string = changelog_header.lstrip("### ").rstrip(" ###") - parentheses_removed_string = remove_parentheses_from_string(hash_removed_string) + parentheses_removed_string = remove_string_inside_parentheses(hash_removed_string) words = parentheses_removed_string.strip().split(" ") if len(words) != 2: raise ValueError("Two words should be included in striped version header") From 25d7ebb0953fdfc3ac04ef7e09a44a00c44a2940 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 7 Apr 2021 17:56:20 +0300 Subject: [PATCH 031/106] Add checkout for citus project for testing --- python/tests/test_prepare_release.py | 30 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index 3fd26b83..f269ac10 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -1,9 +1,11 @@ import shutil import unittest +import os import pathlib2 from ..prepare_release import * +from ..common_tool_methods import * github_token = os.getenv("GH_TOKEN") @@ -14,15 +16,19 @@ class PrepareReleaseTestCases(unittest.TestCase): def test_major_release(self): - shutil.copytree(CITUS_PROJECT_TEST_PATH, CITUS_PROJECT_TEST_PATH_COPY) - try: - update_release(github_token=github_token, project_name="citus", project_version="10.1.0", - main_branch="master", - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=CITUS_PROJECT_TEST_PATH_COPY) - # current branch is release-$version number - # check for configure in - # check for - finally: - # shutil.rmtree(CITUS_PROJECT_TEST_PATH_COPY) - run(f"rm -r {CITUS_PROJECT_TEST_PATH_COPY}") + # shutil.copytree(CITUS_PROJECT_TEST_PATH, CITUS_PROJECT_TEST_PATH_COPY) + if not os.path.exists("citus"): + run("git clone https://github.com/citusdata/citus.git") + run("cd citus") + # try: + update_release(github_token=github_token, project_name="citus", project_version="10.1.0", + main_branch="master", + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=CITUS_PROJECT_TEST_PATH_COPY) + + # current branch is release-$version number + # check for configure in + # check for + # finally: + # # shutil.rmtree(CITUS_PROJECT_TEST_PATH_COPY) + # run(f"rm -r {CITUS_PROJECT_TEST_PATH_COPY}") From b35c6ef78dd27fc8800674801bd728231e2344d6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 7 Apr 2021 18:33:55 +0300 Subject: [PATCH 032/106] Change directory to execute tests in folder --- python/tests/test_prepare_release.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index f269ac10..cd1e0a91 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -9,7 +9,8 @@ github_token = os.getenv("GH_TOKEN") -TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() +BASE_PATH = pathlib2.Path(__file__).parents[2] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") +TEST_BASE_PATH = f"{BASE_PATH}/citus" CITUS_PROJECT_TEST_PATH = f"{TEST_BASE_PATH}/projects/citus" CITUS_PROJECT_TEST_PATH_COPY = f"{TEST_BASE_PATH}/projects/citus_copy" @@ -19,12 +20,12 @@ def test_major_release(self): # shutil.copytree(CITUS_PROJECT_TEST_PATH, CITUS_PROJECT_TEST_PATH_COPY) if not os.path.exists("citus"): run("git clone https://github.com/citusdata/citus.git") - run("cd citus") + os.chdir("citus") # try: - update_release(github_token=github_token, project_name="citus", project_version="10.1.0", + update_release(github_token=github_token, project_name="citus", project_version="10.2.0", main_branch="master", earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=CITUS_PROJECT_TEST_PATH_COPY) + exec_path=TEST_BASE_PATH) # current branch is release-$version number # check for configure in From b4078c65e80ff5fa7f058452b3afd48e27ab327f Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 7 Apr 2021 21:07:03 +0300 Subject: [PATCH 033/106] Add test suffix into branch name to test the branch --- python/prepare_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/prepare_release.py b/python/prepare_release.py index ef03fea6..f4542ec1 100644 --- a/python/prepare_release.py +++ b/python/prepare_release.py @@ -33,7 +33,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver "UPCOMING_VERSION") upcoming_version_details = get_version_details(upcoming_version) upcoming_minor_version = f'{upcoming_version_details["major"]}.{upcoming_version_details["minor"]}' - release_branch_name = f'release-{project_version_details["major"]}.{project_version_details["minor"]}' + release_branch_name = f'release-test-{project_version_details["major"]}.{project_version_details["minor"]}' g = Github(github_token) repository = g.get_repo(f"{REPO_OWNER}/{project_name}") From 3c95b3d2af84aba3f2962de4489edfc969c41449 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 7 Apr 2021 21:11:02 +0300 Subject: [PATCH 034/106] Fix master update branch name --- python/prepare_release.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/prepare_release.py b/python/prepare_release.py index f4542ec1..99697101 100644 --- a/python/prepare_release.py +++ b/python/prepare_release.py @@ -1,6 +1,7 @@ import os from github import Github +import uuid from .common_tool_methods import * @@ -76,7 +77,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver print(f"OK Changes pushed for {release_branch_name}") print(f"OK {release_branch_name} prepared.") # Increase version number and change upcoming version - upcoming_version_branch = f"master-update-version {datetime.now().timestamp()}" + upcoming_version_branch = f"master-update-version-{uuid.uuid4()}" print(f"Preparing upcoming version branch for the new master with name {upcoming_version_branch}...") # checkout master print(f"Checking out {main_branch}...") From 365dede6389bba124f8957cd38e5338e45452863 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 9 Apr 2021 12:30:19 +0300 Subject: [PATCH 035/106] Add unit tests,fix problems on prepare_release that was caught while unit testing --- python/common_tool_methods.py | 30 ++- python/prepare_release.py | 301 ++++++++++++++++----------- python/test-features.py | 8 +- python/tests/test_prepare_release.py | 118 +++++++++-- 4 files changed, 320 insertions(+), 137 deletions(-) diff --git a/python/common_tool_methods.py b/python/common_tool_methods.py index a343ba84..1a831119 100644 --- a/python/common_tool_methods.py +++ b/python/common_tool_methods.py @@ -109,15 +109,43 @@ def get_pr_issues_by_label(prs: List[PullRequest.PullRequest], label_name: str): return filtered_prs -def replace_line_in_file(file: str, match_regex: str, replace_str: str): +def has_file_include_line(base_path: str, relative_file_path: str, line_content: str) -> bool: + with open(f"{base_path}/{relative_file_path}", "r") as reader: + content = reader.read() + lines = content.splitlines() + found = False + for line in lines: + if line == line_content: + found = True + break + return found + + +def line_count_in_file(base_path: str, relative_file_path: str, line_content: str) -> int: + with open(f"{base_path}/{relative_file_path}", "r") as reader: + content = reader.read() + lines = content.splitlines() + counter = 0 + for line in lines: + if line == line_content: + counter = counter + 1 + + return counter + + +def replace_line_in_file(file: str, match_regex: str, replace_str: str) -> bool: with open(file, "r") as reader: file_content = reader.read() lines = file_content.splitlines() line_counter = 0 + has_match = False for line in lines: if re.match(match_regex, line): + has_match = True lines[line_counter] = replace_str line_counter = line_counter + 1 edited_content = str_array_to_str(lines) with open(file, "w") as writer: writer.write(edited_content) + + return has_match diff --git a/python/prepare_release.py b/python/prepare_release.py index 99697101..9ea9d9b8 100644 --- a/python/prepare_release.py +++ b/python/prepare_release.py @@ -1,9 +1,11 @@ import os +import uuid +from typing import Tuple from github import Github -import uuid from .common_tool_methods import * +from enum import Enum MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" CITUS_CONTROL = "src/backend/distributed/citus.control" @@ -11,15 +13,28 @@ CONFIG_PY = "src/test/regress/upgrade/config.py" DISTRIBUTED_DIR_PATH = "src/backend/distributed" CONFIGURE_IN = "configure.in" +CONFIGURE = "configure" CITUS_CONTROL_SEARCH_PATTERN = r"^default_version*" -MULT_EXT_SEARCH_PATTERN = r"^\d{1,2}\.\d{1,2}devel$" +MULT_EXT_MAJOR_SEARCH_PATTERN = r"^\s*\d{1,2}\.\d{1,2}devel$" +MULT_EXT_PATCH_SEARCH_PATTERN = r"^\s*\d{1,2}\.\d{1,2}.\d{1,2}$" + CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" REPO_OWNER = "citusdata" +class ResourceStatus(Enum): + INITIAL = 1 + RELEASE_BRANCH_LOCAL = 2 + RELEASE_BRANCH_REMOTE = 3 + UPCOMING_VERSION_LOCAL = 4 + UPCOMING_VERSION_REMOTE = 5 + PULL_REQUEST_CREATED = 6 + + def update_release(github_token: str, project_name: str, project_version: is_version(str), main_branch: str, - earliest_pr_date: datetime, exec_path: str): + earliest_pr_date: datetime, exec_path: str, is_test: bool = False, + cherry_pick_enabled: bool = False) -> Tuple[str, str, str, ResourceStatus]: multi_extension_sql_path = f"{exec_path}/{MULTI_EXTENSION_SQL}" citus_control_file_path = f"{exec_path}/{CITUS_CONTROL}" multi_extension_out_path = f"{exec_path}/{MULTI_EXTENSION_OUT}" @@ -27,178 +42,224 @@ def update_release(github_token: str, project_name: str, project_version: is_ver config_py_path = f"{exec_path}/{CONFIG_PY}" distributed_dir_path = f"{exec_path}/{DISTRIBUTED_DIR_PATH}" + resource_status = ResourceStatus.INITIAL + project_version_details = get_version_details(project_version) default_upcoming_version = f'{project_version_details["major"]}.{project_version_details["minor"]}.' \ f'{str(int(project_version_details["patch"]) + 1)}' upcoming_version = default_upcoming_version if os.getenv("UPCOMING_VERSION") is None else os.getenv( "UPCOMING_VERSION") upcoming_version_details = get_version_details(upcoming_version) - upcoming_minor_version = f'{upcoming_version_details["major"]}.{upcoming_version_details["minor"]}' - release_branch_name = f'release-test-{project_version_details["major"]}.{project_version_details["minor"]}' + upcoming_patch_version = f'{upcoming_version_details["major"]}.{upcoming_version_details["minor"]}' + devel_version = f"{upcoming_patch_version}devel" + release_branch_name = f'release-{project_version_details["major"]}.{project_version_details["minor"]}' + release_branch_name = f"{release_branch_name}-test" if is_test else release_branch_name g = Github(github_token) repository = g.get_repo(f"{REPO_OWNER}/{project_name}") - + newly_created_sql_file = "" + upcoming_version_branch = f"master-update-version-{uuid.uuid4()}" # if major release if is_major_release(project_version): - print(f"{project_version} is a major release. Executing Major release flow ") + print(f"### {project_version} is a major release. Executing Major release flow### ") # create release-X-Y branch - print(f"Preparing {release_branch_name}...") + print(f"### Preparing {release_branch_name}...### ") # checkout master - print(f"Checking out {main_branch}...") + print(f"### Checking out {main_branch}...### ") run(f"git checkout {main_branch}") run(f"git pull") - print(f"OK {main_branch} checked out and pulled") + print(f"### OK {main_branch} checked out and pulled### ") # create release branch in release-X.Y format - print(f"Creating release branch with name {release_branch_name}...") + print(f"### Creating release branch with name {release_branch_name}...### ") run(f'git checkout -b {release_branch_name}') - print(f"OK {release_branch_name} created") + print(f"### OK {release_branch_name} created### ") + resource_status = ResourceStatus.RELEASE_BRANCH_LOCAL # change version info in configure.in file - print(f"Updating version on file {configure_in_path}...") - replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{project_version}])") - print(f"OK {configure_in_path} file is updated with project version {project_version}.") - # update_configure_in(project_version) + print(f"### Updating version on file {configure_in_path}...### ") + if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, + f"AC_INIT([Citus], [{project_version}])"): + raise ValueError(f"{configure_in_path} does not have match for version") + print(f"### OK {configure_in_path} file is updated with project version {project_version}.### ") # execute "autoconf -f" - print(f"Executing autoconf -f command...") + print(f"### Executing autoconf -f command...### ") run("autoconf -f") - print(f"OK autoconf -f executed.") + print(f"### OK autoconf -f executed.### ") # change version info in multi_extension.out - print(f"Updating version on file {multi_extension_out_path}...") - replace_line_in_file(multi_extension_out_path, MULT_EXT_SEARCH_PATTERN, - f"{project_version}") - print(f"OK {multi_extension_out_path} file is updated with project version {project_version}.") + print(f"### Updating version on file {multi_extension_out_path}...### ") + if not replace_line_in_file(multi_extension_out_path, MULT_EXT_MAJOR_SEARCH_PATTERN, + f" {project_version}"): + raise ValueError(f"{multi_extension_out_path} does not have match for version") + print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.### ") # commit all changes - print(f"Committing changes for branch {release_branch_name}...") - run(f' git commit -am "Bump {project_name} version to {project_version} "') - print(f"OK Changes committed for {release_branch_name}") + print(f"### Committing changes for branch {release_branch_name}...### ") + run(f' git commit -a -m "Bump {project_name} version to {project_version} "') + print(f"### OK Changes committed for {release_branch_name}### ") # push release branch (No PR creation!!!) - print(f"Pushing changes for {release_branch_name} into remote origin...") - run(f"git push --set-upstream origin {release_branch_name}") - print(f"OK Changes pushed for {release_branch_name}") - print(f"OK {release_branch_name} prepared.") + if not is_test: + print(f"### Pushing changes for {release_branch_name} into remote origin...### ") + run(f"git push --set-upstream origin {release_branch_name}") + resource_status = ResourceStatus.RELEASE_BRANCH_REMOTE + print(f"### OK Changes pushed for {release_branch_name}### ") + print(f"### OK {release_branch_name} prepared.### ") + # Increase version number and change upcoming version - upcoming_version_branch = f"master-update-version-{uuid.uuid4()}" - print(f"Preparing upcoming version branch for the new master with name {upcoming_version_branch}...") + + print(f"### Preparing upcoming version branch for the new master with name {upcoming_version_branch}...### ") # checkout master - print(f"Checking out {main_branch}...") + print(f"### Checking out {main_branch}...### ") run(f"git checkout {main_branch}") - print(f"OK {main_branch} checked out.") + print(f"### OK {main_branch} checked out.### ") # create master-update-version-$curtime branch - print(f"Checking out {upcoming_version_branch}...") + print(f"### Checking out {upcoming_version_branch}...### ") run(f"git checkout -b {upcoming_version_branch}") - print(f"{upcoming_version_branch} checked out.") + print(f"### {upcoming_version_branch} checked out.### ") + resource_status = ResourceStatus.UPCOMING_VERSION_LOCAL # update version info with upcoming version on configure.in - print(f"Updating {configure_in_path} file with the upcoming version {upcoming_version}...") - replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{upcoming_version}])") - print(f"U{configure_in_path} file updated with the upcoming version {upcoming_version}...") + print(f"### Updating {configure_in_path} file with the upcoming version {devel_version}...### ") + if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, + f"AC_INIT([Citus], [{devel_version}])"): + raise ValueError(f"{configure_in_path} does not have match for version") + print(f"### {configure_in_path} file updated with the upcoming version {devel_version}...### ") # update version info with upcoming version on config.py - print(f"Updating {config_py_path} file with the upcoming version {upcoming_minor_version}...") - replace_line_in_file(config_py_path, "^MASTER_VERSION =*", - f'MASTER_VERSION = {upcoming_minor_version}') - print(f"U{config_py_path} file updated with the upcoming version {upcoming_minor_version}...") + print(f"### Updating {config_py_path} file with the upcoming version {upcoming_patch_version}...### ") + if not replace_line_in_file(config_py_path, "^MASTER_VERSION =*", + f"MASTER_VERSION = '{upcoming_patch_version}'"): + raise ValueError(f"{config_py_path} does not have match for version") + print(f"### {config_py_path} file updated with the upcoming version {upcoming_patch_version}...### ") # execute autoconf -f - print(f"Executing autoconf -f command...") + print(f"### Executing autoconf -f command...### ") run("autoconf -f") - print(f"OK autoconf -f executed.") + print(f"### OK autoconf -f executed.### ") # update version info with upcoming version on multiextension.out # TODO May add a special version descriptor to address lines directly to be replaced - print(f"Updating {multi_extension_out_path} file with the upcoming version {upcoming_version}...") - replace_line_in_file(multi_extension_out_path, MULT_EXT_SEARCH_PATTERN, - f"{upcoming_version}") - print(f"U{multi_extension_out_path} file updated with the upcoming version {upcoming_version}...") + print(f"### Updating {multi_extension_out_path} file with the upcoming version {devel_version}...### ") + if not replace_line_in_file(multi_extension_out_path, MULT_EXT_MAJOR_SEARCH_PATTERN, + f" {devel_version}"): + raise ValueError(f"{multi_extension_out_path} does not have match for version") + print(f"### OK {multi_extension_out_path} file updated with the upcoming version {devel_version}...### ") # get current schema version from citus.control - print(f"Reading current schema version from {citus_control_file_path}...") - current_schema_version = "" - with open(citus_control_file_path, "r") as cc_reader: - cc_file_content = cc_reader.read() - cc_lines = cc_file_content.splitlines() - for cc_line in cc_lines: - if re.match(CITUS_CONTROL_SEARCH_PATTERN, cc_line): - current_schema_version = cc_line - + print(f"### Reading current schema version from {citus_control_file_path}...### ") + current_schema_version = get_current_schema_version(citus_control_file_path) if len(current_schema_version) == 0: raise ValueError("Version info could not be found in citus.control file") - print(f"OK Current schema version is {current_schema_version}") + print(f"### OK Current schema version is {current_schema_version}### ") # find current schema version info and update it with upcoming version in multi_extension.sql file print( - f"Updating schema version {current_schema_version} on {multi_extension_sql_path} " - f"file with the upcoming version {upcoming_version}...") - replace_line_in_file(multi_extension_sql_path, - f"ALTER EXTENSION citus UPDATE TO {current_schema_version}", - f"ALTER EXTENSION citus UPDATE TO {upcoming_minor_version}") - print(f"OK Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}") + f"### Updating schema version {current_schema_version} on {multi_extension_sql_path} " + f"file with the upcoming version {upcoming_version}...### ") + # TODO Append instead of replace may require + if not replace_line_in_file(multi_extension_sql_path, + f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", + f"ALTER EXTENSION citus UPDATE TO '{upcoming_patch_version}-1';"): + raise ValueError(f"{multi_extension_sql_path} does not have match for version") + print(f"### OK Current schema version updated on {multi_extension_sql_path} to {upcoming_patch_version}### ") # find current schema version info and update it with upcoming version in multi_extension.out file print( - f"Updating schema version {current_schema_version} on {multi_extension_out_path} " - f"file with the upcoming version {upcoming_version}...") - replace_line_in_file(multi_extension_out_path, - f"ALTER EXTENSION citus UPDATE TO {current_schema_version}", - f"ALTER EXTENSION citus UPDATE TO {upcoming_minor_version}") - print(f"OK Current schema version updated on {multi_extension_out_path} to {upcoming_minor_version}") + f"### Updating schema version {current_schema_version} on {multi_extension_out_path} " + f"file with the upcoming version {upcoming_version}...### ") + if not replace_line_in_file(multi_extension_out_path, + f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", + f"ALTER EXTENSION citus UPDATE TO '{upcoming_patch_version}-1';"): + raise ValueError(f"{multi_extension_out_path} does not have match for version") + print(f"### OK Current schema version updated on {multi_extension_out_path} to {upcoming_patch_version}### ") # create new sql file with the name ">./src/backend/distributed/citus--$current_schema_version-- # $upcoming_minor_version-1.sql" - newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" - print(f"Creating file {newly_created_sql_file}...") + newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_patch_version}-1.sql" + print(f"### Creating file {newly_created_sql_file}...### ") with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: - content = f"/* citus--{current_schema_version}--{upcoming_minor_version}-1 */" + content = f"/* citus--{current_schema_version}--{upcoming_patch_version}-1 */" content = content + "\n\n" - content = content + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" + content = content + f"-- bump version to {upcoming_patch_version}-1" + "\n\n" f_writer.write(content) - print(f"{newly_created_sql_file} created.") + run(f"git add {distributed_dir_path}/{newly_created_sql_file}") + print(f"### OK {newly_created_sql_file} created.") # change version in citus.control file - print(f"Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}...") - replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, - f"default_version={upcoming_minor_version}-1") - print(f"U{citus_control_file_path} file updated with the upcoming version {upcoming_minor_version}...") + print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_patch_version}...### ") + if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, + f"default_version = '{upcoming_patch_version}-1'"): + raise ValueError(f"{citus_control_file_path} does not have match for version") + print(f"### OK{citus_control_file_path} file updated with the upcoming version {upcoming_patch_version}...### ") # commit and push changes on master-update-version-$curtime branch - print(f"Committing changes for branch {upcoming_version_branch}...") - run(f'git commit -am "{project_name} version to {project_version}"') - print(f"OK Changes committed for {upcoming_version_branch}") - print(f"Pushing changes for {upcoming_version_branch} into remote origin...") - run(f"git push --set-upstream origin {upcoming_version_branch}") - print(f"OK Changes pushed for {upcoming_version_branch}") + print(f"### Committing changes for branch {upcoming_version_branch}...### ") + run(f'git commit -a -m "Bump {project_name} version to {project_version}"') + print(f"### OK Changes committed for {upcoming_version_branch}") + if not is_test: + print(f"Pushing changes for {upcoming_version_branch} into remote origin...### ") + run(f"git push --set-upstream origin {upcoming_version_branch}") + resource_status = ResourceStatus.UPCOMING_VERSION_REMOTE + print(f"### OK Changes pushed for {upcoming_version_branch}### ") # create pull request - print(f"Creating pull request for {upcoming_version_branch}") - pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, - head=upcoming_version_branch, body="") - print(f"OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}") + if not is_test: + print(f"### Creating pull request for {upcoming_version_branch}### ") + pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, + head=upcoming_version_branch, body="") + print(f"### OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}### ") + resource_status = ResourceStatus.PULL_REQUEST_CREATED else: - print(f"{project_version} is a patch release. Executing Patch release flow ") + print(f"### {project_version} is a patch release. Executing Patch release flow ### ") # checkout release branch in release-X.Y format - print(f"Checking out {release_branch_name}...") + print(f"### Checking out {release_branch_name}...### ") run(f'git checkout {release_branch_name}') - print(f"{release_branch_name} checked out") - # list all pr's with backport labels - print(f"Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}") - all_related_prs = get_prs(repository, earliest_pr_date, main_branch) - # get commits for selected prs with backport label - prs_with_backport = get_pr_issues_by_label(all_related_prs, "backport") - print(f"OK {len(prs_with_backport)} PRs with backport label found. PR list is as below") - for pr in prs_with_backport: - print(f"No:{pr.number} Title:{pr.title}\n") - - # cherrypick all commits with backport label - - print(f"Cherry-picking PRs on {release_branch_name}...") - cherry_pick_prs(prs_with_backport) - print(f"OK Cherry pick completed for all PRs on branch {release_branch_name}") - # change version info in configure.in file + print(f"### OK {release_branch_name} checked out### ") - replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{project_version}])") + # change version info in configure.in file + print(f"### Updating {configure_in_path} file with the project version {project_version}...### ") + if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, + f"AC_INIT([Citus], [{project_version}])"): + raise ValueError(f"{configure_in_path} does not have match for version") + print(f"### OK {configure_in_path} file is updated with project version {project_version}.### ") # execute "auto-conf " (Not with f flag!!!) - run("autoconf") + print(f"### Executing autoconf command...### ") + run("autoconf -f") + print(f"### OK autoconf executed.### ") # change version info in multi_extension.out - print(f"Updating {multi_extension_out_path} file with the project version {project_version}...") - replace_line_in_file(multi_extension_out_path, MULT_EXT_SEARCH_PATTERN, - f"{project_version}") - print(f"OK {multi_extension_out_path} file is updated with project version {project_version}.") - # commit all-changes - print(f"Committing changes for branch {release_branch_name}...") - run(f'git commit -am "{project_name} version to {project_version}"') - print(f"OK Changes committed for {release_branch_name}") + print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...### ") + if not replace_line_in_file(multi_extension_out_path, MULT_EXT_PATCH_SEARCH_PATTERN, + f" {project_version}"): + raise ValueError(f"{multi_extension_out_path} does not have match for version") + print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.### ") + + if cherry_pick_enabled: + # list all pr's with backport labels + print( + f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}### ") + all_related_prs = get_prs(repository, earliest_pr_date, main_branch) + # get commits for selected prs with backport label + prs_with_backport = get_pr_issues_by_label(all_related_prs, "backport") + print(f"### OK {len(prs_with_backport)} PRs with backport label found. PR list is as below### ") + for pr in prs_with_backport: + print(f"No:{pr.number} Title:{pr.title}\n") + # cherrypick all commits with backport label + + print(f"Cherry-picking PRs on {release_branch_name}...") + cherry_pick_prs(prs_with_backport) + print(f"OK Cherry pick completed for all PRs on branch {release_branch_name}") + # commit all-changes + + print(f"### Committing changes for branch {release_branch_name}...### ") + run(f'git commit -a -m "{project_name} version to {project_version}"') + print(f"### OK Changes committed for {release_branch_name}### ") # create and push release-$minor_version-push-$curTime branch - print(f"Pushing changes for {release_branch_name} into remote origin...") - run(f"git push --set-upstream origin {release_branch_name}") - print(f"OK Changes pushed for {release_branch_name}") + if not is_test: + print(f"### Pushing changes for {release_branch_name} into remote origin...### ") + run(f"git push --set-upstream origin {release_branch_name}") + resource_status = ResourceStatus.UPCOMING_VERSION_REMOTE + print(f"### OK Changes pushed for {release_branch_name}### ") + return release_branch_name, upcoming_version_branch, f"{DISTRIBUTED_DIR_PATH}/{newly_created_sql_file}", \ + resource_status + + +def get_current_schema_version(citus_control_file_path): + current_schema_version = "" + with open(citus_control_file_path, "r") as cc_reader: + cc_file_content = cc_reader.read() + cc_lines = cc_file_content.splitlines() + for cc_line in cc_lines: + if re.match(CITUS_CONTROL_SEARCH_PATTERN, cc_line): + line_parts = cc_line.split("=") + if len(line_parts) > 0: + current_schema_version = line_parts[1] + current_schema_version = current_schema_version.strip(" '") + return current_schema_version diff --git a/python/test-features.py b/python/test-features.py index aa619943..9dc3f48a 100644 --- a/python/test-features.py +++ b/python/test-features.py @@ -5,6 +5,10 @@ import subprocess import uuid -g = Github("6df6f95934896fd559e4b2f7eb630c49f8dad560") -repository = g.get_repo(f"citusdata/citus") +# g = Github("6df6f95934896fd559e4b2f7eb630c49f8dad560") +# repository = g.get_repo(f"citusdata/citus") +# +# pr_result = repository.create_pull(title=f"Bump Citus to 10.2.1", base="master", +# head="master-update-version-9b5990ba-5288-44cc-ab38-d31c7ec22667", body="") +str_test = "[xsds" \ No newline at end of file diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index cd1e0a91..551f2d98 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -1,11 +1,8 @@ -import shutil import unittest -import os import pathlib2 from ..prepare_release import * -from ..common_tool_methods import * github_token = os.getenv("GH_TOKEN") @@ -16,20 +13,113 @@ class PrepareReleaseTestCases(unittest.TestCase): - def test_major_release(self): - # shutil.copytree(CITUS_PROJECT_TEST_PATH, CITUS_PROJECT_TEST_PATH_COPY) + def setUp(self): + if os.path.exists("citus"): + run("rm -r citus") + self.initialize_env() + + def initialize_env(self): + print("Current Directory:"+os.getcwd()) if not os.path.exists("citus"): run("git clone https://github.com/citusdata/citus.git") + + def test_major_release(self): + self.initialize_env() + os.chdir("citus") + release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( + github_token=github_token, + project_name="citus", + project_version="10.2.0", + main_branch="master", + earliest_pr_date=datetime.strptime( + '2021.03.25 00:00', + '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, + is_test=True) + # run(f"git checkout {release_branch}") + + # newly_created_sql_file = f"{DISTRIBUTED_DIR_PATH}/citus--10.1-1--10.2-1.sql" + + run(f"git checkout {release_branch}") + + self.assertTrue(has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.0")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.0])")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.0'")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, + "\`configure' configures Citus 10.2.0 to adapt to many kinds of systems.")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2.0:";;')) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'")) + + run(f"git checkout {upcoming_version_branch}") + + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'")) + self.assertTrue( + has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, "ALTER EXTENSION citus UPDATE TO '10.2-1';")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel")) + self.assertEqual(2, line_count_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, + "ALTER EXTENSION citus UPDATE TO '10.2-1';")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, + "\`configure' configures Citus 10.2devel to adapt to many kinds of systems.")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;')) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'")) + self.assertTrue(os.path.exists(f"{TEST_BASE_PATH}/{newly_created_sql_file}")) + # + + if resource_status in (ResourceStatus.RELEASE_BRANCH_LOCAL, ResourceStatus.RELEASE_BRANCH_REMOTE, + ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, + ResourceStatus.PULL_REQUEST_CREATED): + run(f"git branch -D {release_branch} ") + elif resource_status in (ResourceStatus.RELEASE_BRANCH_REMOTE, + ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, + ResourceStatus.PULL_REQUEST_CREATED): + run(f"git push origin --delete {release_branch}") + elif resource_status in (ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, + ResourceStatus.PULL_REQUEST_CREATED): + run(f"git branch -D {upcoming_version_branch} ") + elif resource_status in (ResourceStatus.UPCOMING_VERSION_REMOTE, + ResourceStatus.PULL_REQUEST_CREATED): + run(f"git push origin --delete {release_branch}") + self.clear_env() + + def test_patch_release(self): + self.initialize_env() + print(os.getcwd()) os.chdir("citus") - # try: + update_release(github_token=github_token, project_name="citus", project_version="10.2.0", main_branch="master", earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH) - - # current branch is release-$version number - # check for configure in - # check for - # finally: - # # shutil.rmtree(CITUS_PROJECT_TEST_PATH_COPY) - # run(f"rm -r {CITUS_PROJECT_TEST_PATH_COPY}") + exec_path=TEST_BASE_PATH, is_test=True) + + update_release(github_token=github_token, project_name="citus", project_version="10.2.1", + main_branch="master", + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, is_test=True) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, + "\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.")) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2.1:";;')) + self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'")) + + self.clear_env() + + # def tearDown(self): + # self.clear_env() + + def clear_env(self): + + if os.path.exists("../citus"): + os.chdir("..") + run("chmod -R 777 citus") + run("sudo rm -rf citus") From ec33dba9472e8b69df1f8176d7e3ee156d7ddacf Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 9 Apr 2021 12:51:27 +0300 Subject: [PATCH 036/106] Remove unnecessary local test file --- python/test-features.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 python/test-features.py diff --git a/python/test-features.py b/python/test-features.py deleted file mode 100644 index 9dc3f48a..00000000 --- a/python/test-features.py +++ /dev/null @@ -1,14 +0,0 @@ -from github import Github, Repository, PullRequest -from datetime import datetime - -from typing import List -import subprocess -import uuid - -# g = Github("6df6f95934896fd559e4b2f7eb630c49f8dad560") -# repository = g.get_repo(f"citusdata/citus") -# -# pr_result = repository.create_pull(title=f"Bump Citus to 10.2.1", base="master", -# head="master-update-version-9b5990ba-5288-44cc-ab38-d31c7ec22667", body="") - -str_test = "[xsds" \ No newline at end of file From a7377461929e1dd65cc4f4517360a2c40df6720e Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 9 Apr 2021 14:20:17 +0300 Subject: [PATCH 037/106] Add Prepare release into tools test pipeline Remove unnecessary logs --- .github/workflows/tool-tests.yml | 18 ++++-------------- python/tests/test_prepare_release.py | 4 ++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 4dcc1711..fbc9bd76 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -1,38 +1,28 @@ -# This is a basic workflow to help you get started with Actions - name: Tool Tests env: GH_TOKEN: ${{ secrets.GH_TOKEN }} - -# Controls when the action will run. on: - # Triggers the workflow on push or pull request events but only for the develop branch push: branches: "**" pull_request: branches: "**" - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" + build: - # The type of runner that the job will run on runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - name: install requirements run: python -m pip install -r python/requirements.txt - - name: Execute Unit tests + - name: Unit tests for "Update Package Properties" run: python -m unittest $(pwd)/python/tests/test_update_package_properties.py + - name: Unit tests for "Prepare Release" + run: python -m unittest $(pwd)/python/tests/test_prepare_release.py diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index 551f2d98..1e66baa3 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -19,7 +19,7 @@ def setUp(self): self.initialize_env() def initialize_env(self): - print("Current Directory:"+os.getcwd()) + # print("Current Directory:"+os.getcwd()) if not os.path.exists("citus"): run("git clone https://github.com/citusdata/citus.git") @@ -90,7 +90,7 @@ def test_major_release(self): def test_patch_release(self): self.initialize_env() - print(os.getcwd()) + # print(os.getcwd()) os.chdir("citus") update_release(github_token=github_token, project_name="citus", project_version="10.2.0", From 0d7ba17579000d486edac1971fab6f9e6e2a55e6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 9 Apr 2021 14:26:14 +0300 Subject: [PATCH 038/106] Add git config info to pull git repo for prepare_release tests --- .github/workflows/tool-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index fbc9bd76..73b62cd5 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -17,6 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + + - name: Define git credentails + run: git config --global user.email "gindibay@microsoft.com"&& git config --global user.name "Gurkan Indibay" - name: install requirements run: python -m pip install -r python/requirements.txt - name: Unit tests for "Update Package Properties" From fecb105d81e4a3c695fc62ec570d681d56057416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 12 Apr 2021 06:42:53 +0300 Subject: [PATCH 039/106] Simplified git ignore of pycache Co-authored-by: Jelte Fennema --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1189a22b..5d6793bc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,4 @@ # generated man pages *.1 -/python/__pycache__/ -/python/tests/__pycache__/ +__pycache__/ From eb6ea07caaf844d83e6e048f3077bfce234de5e1 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 06:53:16 +0300 Subject: [PATCH 040/106] Remove useless files Set default variables on update_package_properties.sh to prevent "unbound variable" error --- bash/execute_unit_tests.sh | 1 - bash/execute_update_properties.sh | 4 ---- bash/update_package_properties.sh | 14 +++++++------- 3 files changed, 7 insertions(+), 12 deletions(-) delete mode 100644 bash/execute_unit_tests.sh delete mode 100644 bash/execute_update_properties.sh diff --git a/bash/execute_unit_tests.sh b/bash/execute_unit_tests.sh deleted file mode 100644 index a9bf588e..00000000 --- a/bash/execute_unit_tests.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/bash/execute_update_properties.sh b/bash/execute_update_properties.sh deleted file mode 100644 index 995af6d9..00000000 --- a/bash/execute_update_properties.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -python ../../python/update_package_properties.py --gh_token "${GITHUB_TOKEN}" --prj_name "${PRJ_NAME}" --prj_ver "${PRJ_VER}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ ---fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S &z')" --exec-path "$(pwd)" &>2 - diff --git a/bash/update_package_properties.sh b/bash/update_package_properties.sh index bbaeec51..3fb206a1 100644 --- a/bash/update_package_properties.sh +++ b/bash/update_package_properties.sh @@ -10,13 +10,13 @@ set -euxo pipefail #MICROSOFT_EMAIL="gindibay@gmail.com" #NAME="Gurkan Indibay" -[ -z "${PRJ_NAME}" ] && echo "PRJ_NAME should be non-empty value" && exit 1 -[ -z "${PRJ_VER}" ] && echo "PRJ_VER should be non-empty value" && exit 1 -[ -z "${TAG_NAME}" ] && echo "TAG_NAME should be non-empty value" && exit 1 -[ -z "${FANCY}" ] && echo "FANCY should be non-empty value" && exit 1 -[ -z "${FANCY_VERSION_NO}" ] && echo "FANCY_VERSION_NO should be non-empty value" && exit 1 -[ -z "${MICROSOFT_EMAIL}" ] && echo "MICROSOFT_EMAIL should be non-empty value" && exit 1 -[ -z "${NAME}" ] && echo "NAME should be non-empty value" && exit 1 +[ -z "${PRJ_NAME:-}" ] && echo "PRJ_NAME should be non-empty value" && exit 1 +[ -z "${PRJ_VER:-}" ] && echo "PRJ_VER should be non-empty value" && exit 1 +[ -z "${TAG_NAME:-}" ] && echo "TAG_NAME should be non-empty value" && exit 1 +[ -z "${FANCY:-}" ] && echo "FANCY should be non-empty value" && exit 1 +[ -z "${FANCY_VERSION_NO:-}" ] && echo "FANCY_VERSION_NO should be non-empty value" && exit 1 +[ -z "${MICROSOFT_EMAIL:-}" ] && echo "MICROSOFT_EMAIL should be non-empty value" && exit 1 +[ -z "${NAME:-}" ] && echo "NAME should be non-empty value" && exit 1 main_branch_name=$(git branch --show-current) From 3e039cfead41d246652aef82476f2a1c889e0f46 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 08:46:58 +0300 Subject: [PATCH 041/106] Removed useless comments on pipeline file Simplify string join blocks on update_package_properties.py --- .github/workflows/tool-tests.yml | 18 +----------------- python/update_package_properties.py | 17 ++++++----------- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 4dcc1711..daafd6e8 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -1,38 +1,22 @@ -# This is a basic workflow to help you get started with Actions - name: Tool Tests env: GH_TOKEN: ${{ secrets.GH_TOKEN }} - -# Controls when the action will run. on: - # Triggers the workflow on push or pull request events but only for the develop branch push: branches: "**" - pull_request: - branches: "**" - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" build: - # The type of runner that the job will run on runs-on: ubuntu-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - name: install requirements run: python -m pip install -r python/requirements.txt - name: Execute Unit tests - run: python -m unittest $(pwd)/python/tests/test_update_package_properties.py - - - + run: python -m unittest python/tests/test_update_package_properties.py \ No newline at end of file diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 2a48b2fe..6aa701c8 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -103,7 +103,7 @@ def get_last_changelog_content(all_changelog_content: str) -> str: def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: second_changelog_index = find_nth_overlapping_line_by_regex(all_changelog_content, "^[a-zA-Z]", 2) lines = all_changelog_content.splitlines() - changelogs = "".join([str(elem) + "\n" for elem in lines[:second_changelog_index - 1]]) + changelogs = "\n".join(lines[:second_changelog_index - 1]) + "\n" if len(lines) < 1: raise ValueError("At least one line should be in changelog") return changelogs @@ -212,17 +212,12 @@ def get_rpm_changelog_from_actual_changelog(project_name: str, project_version: spec_file_path: str): rpm_changelog_history = get_rpm_changelog_history(project_name, spec_file_path) changelog_histories = rpm_changelog_history.splitlines() - rpm_changelog = "" - if len(changelog_histories) < 1 or not (project_version in changelog_histories[1]): - lines = latest_changelog.splitlines() - lines[0] = get_rpm_header(project_name, microsoft_email, name_surname, project_version, fancy, - fancy_version_number, date.today()) - rpm_changelog = rpm_changelog.join([str(elem) + "\n" for elem in lines]) - rpm_changelog = f"{rpm_changelog}{rpm_changelog_history}" - else: - rpm_changelog = rpm_changelog_history + if len(changelog_histories) > 1 and project_version in changelog_histories[1]: + return rpm_changelog_history - return rpm_changelog + header = get_rpm_header(project_name, microsoft_email, name_surname, project_version, fancy, + fancy_version_number, date.today()) + return '\n'.join([header, latest_changelog, rpm_changelog_history])+"\n" def convert_citus_changelog_into_rpm_changelog(project_name: str, project_version: str, microsoft_email: str, From d9e267cbe12d7ce3a1e1b290ba633a0676eb7af5 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 08:48:24 +0300 Subject: [PATCH 042/106] Add missing emptylines --- python/templates/pkgvars.tmpl | 2 +- python/templates/project.spec.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/templates/pkgvars.tmpl b/python/templates/pkgvars.tmpl index 7e41782d..dfa77e69 100644 --- a/python/templates/pkgvars.tmpl +++ b/python/templates/pkgvars.tmpl @@ -3,4 +3,4 @@ pkgdesc='Citus (Open-Source)' pkglatest={{version}} releasepg=11,12,13 nightlypg=12,13 -versioning=fancy \ No newline at end of file +versioning=fancy diff --git a/python/templates/project.spec.tmpl b/python/templates/project.spec.tmpl index 2edbe668..ae0ace57 100644 --- a/python/templates/project.spec.tmpl +++ b/python/templates/project.spec.tmpl @@ -79,4 +79,4 @@ echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list %doc %{pginstdir}/doc/extension/README-%{sname}.md %changelog -{{changelog}} \ No newline at end of file +{{changelog}} From 4fa9f9d7022ab2e1c127b8e9eda64694ff091655 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 20:46:21 +0300 Subject: [PATCH 043/106] Transform long methpod parameters into class --- .../tests/test_update_package_properties.py | 42 +++-- python/update_package_properties.py | 154 ++++++++++++------ 2 files changed, 135 insertions(+), 61 deletions(-) diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index 8541ed93..aaeb7a72 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -1,6 +1,5 @@ import os import unittest -from datetime import datetime from shutil import copyfile from .test_utils import are_strings_equal @@ -58,10 +57,14 @@ def test_prepend_latest_changelog_into_debian_changelog(self): changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" copyfile(refer_file_path, changelog_file_path) latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog( + latest_changelog) try: - prepend_latest_changelog_into_debian_changelog(latest_changelog, PROJECT_VERSION, True, 1, - changelog_file_path, - MICROSOFT_EMAIL, NAME_SURNAME, CHANGELOG_DATE) + prepend_latest_changelog_into_debian_changelog(changelog_param, changelog_file_path) self.verify_prepend_debian_changelog(changelog_file_path) finally: os.remove(changelog_file_path) @@ -71,10 +74,13 @@ def test_prepend_latest_changelog_into_debian_changelog_10_0_3_already_included( changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" copyfile(refer_file_path, changelog_file_path) latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog( + latest_changelog) try: - prepend_latest_changelog_into_debian_changelog(latest_changelog, PROJECT_VERSION, True, 1, - changelog_file_path, - MICROSOFT_EMAIL, NAME_SURNAME, CHANGELOG_DATE) + prepend_latest_changelog_into_debian_changelog(changelog_param, changelog_file_path) self.assertRaises(ValueError) finally: os.remove(changelog_file_path) @@ -88,9 +94,11 @@ def verify_prepend_debian_changelog(self, changelog_file_path): are_strings_equal(expected_content, latest_changelog) def test_convert_citus_changelog_into_rpm_changelog(self): - changelog = convert_citus_changelog_into_rpm_changelog(PROJECT_NAME, PROJECT_VERSION, MICROSOFT_EMAIL, - NAME_SURNAME, True, 1, - CHANGELOG_DATE) + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog("") + changelog = convert_citus_changelog_into_rpm_changelog(changelog_param) with open(f"{TEST_BASE_PATH}/files/verify/rpm_latest_changelog_reference.txt", "r") as reader: content = reader.read() self.assertEqual(content, changelog) @@ -103,8 +111,11 @@ def test_update_rpm_spec(self): templates_path = f"{BASE_PATH}/templates" copyfile(spec_file, spec_file_copy) try: - update_rpm_spec(project_name, PROJECT_VERSION, MICROSOFT_EMAIL, NAME_SURNAME, True, 1, - spec_file, CHANGELOG_DATE, templates_path) + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") + update_rpm_spec(changelog_param, spec_file, templates_path) self.verify_rpm_spec(spec_file_reference, spec_file) finally: copyfile(spec_file_copy, spec_file) @@ -117,8 +128,11 @@ def test_update_rpm_spec_include_10_0_3(self): templates_path = f"{BASE_PATH}/templates" copyfile(spec_file, spec_file_copy) try: - update_rpm_spec(project_name, PROJECT_VERSION, MICROSOFT_EMAIL, NAME_SURNAME, True, 1, - spec_file, CHANGELOG_DATE, templates_path) + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") + update_rpm_spec(changelog_param, spec_file, templates_path) self.assertRaises(ValueError) finally: copyfile(spec_file_copy, spec_file) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 6aa701c8..86806f63 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -46,6 +46,73 @@ def is_project_changelog_header(header: str): f"changelog header is in invalid format. Actual:{header} Expected: ### citus v8.3.3 (March 23, 2021) ### ") +class ChangelogParams: + __latest_changelog: str = "" + __project_version: str = "" + __project_name: str = "" + __fancy: bool = False + __fancy_version_number: int + __microsoft_email: str = "" + __name_surname: str = "" + __changelog_date: datetime = datetime.now() + + def get_latest_changelog(self) -> str: + return self.__latest_changelog + + def set_latest_changelog(self, param: str): + self.__latest_changelog = param + return self + + def get_project_name(self) -> str: + return self.__project_name + + def set_project_name(self, param: str): + self.__project_name = param + return self + + def get_project_version(self) -> str: + return self.__project_version + + def set_project_version(self, param: str): + self.__project_version = param + return self + + def get_fancy(self) -> bool: + return self.__fancy + + def set_fancy(self, param: bool): + self.__fancy = param + return self + + def get_fancy_version_number(self) -> int: + return self.__fancy_version_number + + def set_fancy_version_number(self, param: int): + self.__fancy_version_number = param + return self + + def get_microsoft_email(self) -> str: + return self.__microsoft_email + + def set_microsoft_email(self, param: str): + self.__microsoft_email = param + return self + + def get_name_surname(self) -> str: + return self.__name_surname + + def set_name_surname(self, param: str): + self.__name_surname = param + return self + + def get_changelog_date(self) -> datetime: + return self.__changelog_date + + def set_changelog_date(self, param: datetime): + self.__changelog_date = param + return self + + def get_spec_file_name(project_name: str) -> str: return f"{project_name}.spec" @@ -55,7 +122,8 @@ def get_version_number(version: str, fancy: bool, fancy_release_count: int) -> s return f"{version}{fancy_suffix}" -def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, fancy_release_count: int) -> str: +def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, + fancy_release_count: int) -> str: fancy_suffix = f"-{fancy_release_count}" if fancy else "" return f"{version}.{project_name}{fancy_suffix}" @@ -140,12 +208,12 @@ def get_debian_changelog_header(changelog_header: is_project_changelog_header(st return f"{project_name} ({version_on_changelog}) stable; urgency=low" -def convert_citus_changelog_into_debian_changelog(citus_changelog: str, fancy: bool, - fancy_version_number: int, microsoft_email: str, name_surname: str, - changelog_date) -> str: - lines = citus_changelog.splitlines() - lines[0] = get_debian_changelog_header(lines[0], fancy, fancy_version_number) - lines.append(get_debian_trailer(microsoft_email, name_surname, changelog_date)) +def convert_citus_changelog_into_debian_changelog(changelog_params: ChangelogParams) -> str: + lines = changelog_params.get_latest_changelog().splitlines() + lines[0] = get_debian_changelog_header(lines[0], changelog_params.get_fancy(), + changelog_params.get_fancy_version_number()) + lines.append(get_debian_trailer(changelog_params.get_microsoft_email(), changelog_params.get_name_surname(), + changelog_params.get_changelog_date())) debian_latest_changelog = "" for i in range(len(lines)): append_line = lines[i] if i == 0 or i == len(lines) - 1 else ' ' + lines[i] @@ -153,15 +221,10 @@ def convert_citus_changelog_into_debian_changelog(citus_changelog: str, fancy: b return debian_latest_changelog -def prepend_latest_changelog_into_debian_changelog(latest_changelog: str, project_version: str, fancy: bool, - fancy_version_number: int, changelog_file_path: str, - microsoft_email: str, name_surname: str, - changelog_date: date) -> None: - debian_latest_changelog = convert_citus_changelog_into_debian_changelog(latest_changelog, fancy, - fancy_version_number, microsoft_email, - name_surname, changelog_date) +def prepend_latest_changelog_into_debian_changelog(changelog_param: ChangelogParams, changelog_file_path: str) -> None: + debian_latest_changelog = convert_citus_changelog_into_debian_changelog(changelog_param) with open(changelog_file_path, "r+") as reader: - if not (f"({project_version}" in reader.readline()): + if not (f"({changelog_param.get_project_version()}" in reader.readline()): reader.seek(0, 0) old_changelog = reader.read() changelog = f"{debian_latest_changelog}{old_changelog}" @@ -185,7 +248,7 @@ def update_pkgvars(version: is_version(non_empty(no_whitespaces(non_blank(str))) writer.write(pkgvars_content) -def get_rpm_changelog_history(project_name: str, spec_file_path: str) -> str: +def get_rpm_changelog_history(spec_file_path: str) -> str: with open(spec_file_path, "r") as reader: spec_content = reader.read() changelog_index = spec_content.find("%changelog") @@ -194,11 +257,10 @@ def get_rpm_changelog_history(project_name: str, spec_file_path: str) -> str: return changelog_content -def get_rpm_header(project_name: str, microsoft_email: str, name_surname: str, project_version: str, - is_fancy_versioned: bool, - fancy_version_number: int, changelog_date: date): - formatted_date = changelog_date.strftime("%a %b %d %Y") - return f"* {formatted_date} - {name_surname} <{microsoft_email}> {get_version_number_with_project_name(project_name, project_version, is_fancy_versioned, fancy_version_number)} " +def get_rpm_header(changelog_params: ChangelogParams): + formatted_date = changelog_params.get_changelog_date().strftime("%a %b %d %Y") + return f"* {formatted_date} - {changelog_params.get_name_surname()} <{changelog_params.get_microsoft_email()}> " \ + f"{get_version_number_with_project_name(changelog_params.get_project_name(), changelog_params.get_project_version(), changelog_params.get_fancy(), changelog_params.get_fancy_version_number())} " def get_debian_trailer(microsoft_email: str, name_surname: str, changelog_date: date): @@ -210,46 +272,41 @@ def get_rpm_changelog_from_actual_changelog(project_name: str, project_version: microsoft_email: str, name_surname: str, fancy: bool, fancy_version_number: int, spec_file_path: str): - rpm_changelog_history = get_rpm_changelog_history(project_name, spec_file_path) + rpm_changelog_history = get_rpm_changelog_history(spec_file_path) changelog_histories = rpm_changelog_history.splitlines() if len(changelog_histories) > 1 and project_version in changelog_histories[1]: return rpm_changelog_history header = get_rpm_header(project_name, microsoft_email, name_surname, project_version, fancy, fancy_version_number, date.today()) - return '\n'.join([header, latest_changelog, rpm_changelog_history])+"\n" + return '\n'.join([header, latest_changelog, rpm_changelog_history]) + "\n" -def convert_citus_changelog_into_rpm_changelog(project_name: str, project_version: str, microsoft_email: str, - name_surname: str, fancy: bool, fancy_version_number: int, - changelog_date: date) -> str: - header = get_rpm_header(project_name, microsoft_email, name_surname, project_version, fancy, - fancy_version_number, changelog_date) - rpm_changelog = f"{header.strip()}\n- Update to {project_name.capitalize()} {project_version}" +def convert_citus_changelog_into_rpm_changelog(changelog_params: ChangelogParams) -> str: + header = get_rpm_header(changelog_params) + rpm_changelog = f"{header.strip()}\n- Update to {changelog_params.get_project_name().capitalize()} {changelog_params.get_project_version()}" return rpm_changelog -def update_rpm_spec(project_name: str, project_version: str, microsoft_email: str, - name_surname: str, fancy: bool, fancy_version_number: int, spec_file_name: str, - changelog_date: date, templates_path: str) -> None: +def update_rpm_spec(changelog_param: ChangelogParams, spec_file_name: str, + templates_path: str) -> None: env = get_template_environment(templates_path) - fancy_version_str = get_version_number(project_version, fancy, fancy_version_number) + fancy_version_str = get_version_number(changelog_param.get_project_version(), changelog_param.get_fancy(), + changelog_param.get_fancy_version_number()) template = env.get_template('project.spec.tmpl') - rpm_changelog_history = get_rpm_changelog_history(project_name, spec_file_name) + rpm_changelog_history = get_rpm_changelog_history(spec_file_name) history_lines = rpm_changelog_history.splitlines() - if len(history_lines) > 0 and project_version in history_lines[1]: - raise ValueError(f"{project_version} already exists in rpm spec file") + if len(history_lines) > 0 and changelog_param.get_project_version() in history_lines[1]: + raise ValueError(f"{changelog_param.get_project_version()} already exists in rpm spec file") - latest_changelog = convert_citus_changelog_into_rpm_changelog(project_name, project_version, microsoft_email, - name_surname, fancy, fancy_version_number, - changelog_date) + latest_changelog = convert_citus_changelog_into_rpm_changelog(changelog_param) changelog = f"{latest_changelog}\n\n{rpm_changelog_history}" - content = template.render(version=project_version, fancy_version_str=fancy_version_str, - fancy_version_no=fancy_version_number, changelog=changelog) + content = template.render(version=changelog_param.get_project_version(), fancy_version_str=fancy_version_str, + fancy_version_no=changelog_param.get_fancy_version_number(), changelog=changelog) with open(spec_file_name, "w+") as writer: writer.write(content) @@ -263,12 +320,14 @@ def update_all_changes(github_token: non_empty(non_blank(str)), project_name: no templates_path = f"{BASE_PATH}/templates" update_pkgvars(project_version, fancy, fancy_version_number, templates_path, f"{packaging_path}") latest_changelog = get_changelog_for_tag(github_token, project_name, tag_name) - prepend_latest_changelog_into_debian_changelog(latest_changelog, project_version, fancy, fancy_version_number, - f"{packaging_path}/debian/changelog", microsoft_email, name_surname, - release_date) + changelog_param = ChangelogParams() + changelog_param.set_project_version(project_version).set_project_name(project_name).set_microsoft_email( + microsoft_email).set_name_surname(name_surname).set_fancy_version_number( + fancy_version_number).set_changelog_date(release_date).set_fancy(fancy).set_latest_changelog(latest_changelog) + + prepend_latest_changelog_into_debian_changelog(changelog_param, f"{packaging_path}/debian/changelog") spec_file_name = f"{packaging_path}/{get_spec_file_name(project_name)}" - update_rpm_spec(project_name, project_version, microsoft_email, name_surname, fancy, fancy_version_number, - spec_file_name, release_date, templates_path) + update_rpm_spec(changelog_param, spec_file_name, templates_path) # update_all_changes("93647419346e194ae3094598139d68ebf2263ee0", "citus", "10.0.3", "v10.0.3", True, 1, @@ -292,5 +351,6 @@ def update_all_changes(github_token: non_empty(non_blank(str)), project_name: no exec_date = datetime.strptime(args.date, '%Y.%m.%d %H:%M:%S %z') - update_all_changes(args.gh_token, args.prj_name, args.prj_ver, args.tag_name, args.fancy, int(args.fancy_ver_no), + update_all_changes(args.gh_token, args.prj_name, args.prj_ver, args.tag_name, args.fancy, + int(args.fancy_ver_no), args.email, args.name, exec_date, args.exec_path) From ad2bffc6300728f6df00aa1611ab28cdd1f6d786 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 21:15:01 +0300 Subject: [PATCH 044/106] Use getenv default values handling logic Remove unnecessary comments Change pipeline job name Add new line at the end of some files --- .github/workflows/tool-tests.yml | 4 ++-- python/tests/test_update_package_properties.py | 12 ++++++------ python/upload_to_package_cloud.py | 6 ------ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index daafd6e8..88a47b8e 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - build: + unit_test_execution: runs-on: ubuntu-latest steps: @@ -19,4 +19,4 @@ jobs: - name: install requirements run: python -m pip install -r python/requirements.txt - name: Execute Unit tests - run: python -m unittest python/tests/test_update_package_properties.py \ No newline at end of file + run: python -m unittest python/tests/test_update_package_properties.py diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index aaeb7a72..ae8ba62f 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -6,13 +6,13 @@ from ..update_package_properties import * TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() -BASE_PATH = pathlib2.Path(__file__).parents[1] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") +BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[1]) GITHUB_TOKEN = os.getenv("GH_TOKEN") -PROJECT_VERSION = "10.0.3" if os.getenv("PROJECT_VERSION") is None else os.getenv("PROJECT_VERSION") -TAG_NAME = "v10.0.3" if os.getenv("TAG_NAME") is None else os.getenv("TAG_NAME") -PROJECT_NAME = "citus" if os.getenv("PROJECT_NAME") is None else os.getenv("PROJECT_NAME") -MICROSOFT_EMAIL = "gindibay@microsoft.com" if os.getenv("MICROSOFT_EMAIL") is None else os.getenv("MICROSOFT_EMAIL") -NAME_SURNAME = "Gurkan Indibay" if os.getenv("NAME_SURNAME") is None else os.getenv("NAME_SURNAME") +PROJECT_VERSION = os.getenv("PROJECT_VERSION", default="10.0.3") +TAG_NAME = os.getenv("TAG_NAME", default="v10.0.3") +PROJECT_NAME = os.getenv("PROJECT_NAME", default="citus") +MICROSOFT_EMAIL = os.getenv("MICROSOFT_EMAIL", default="gindibay@microsoft.com") +NAME_SURNAME = os.getenv("NAME_SURNAME", default="Gurkan Indibay") CHANGELOG_DATE = datetime.strptime('Thu, 18 Mar 2021 01:40:08 +0000', '%a, %d %b %Y %H:%M:%S %z') if os.getenv( "CHANGELOG_DATE") is None else datetime.strptime(os.getenv("CHANGELOG_DATE"), '%a, %d %b %Y %H:%M:%S %z') diff --git a/python/upload_to_package_cloud.py b/python/upload_to_package_cloud.py index cee4c94a..90f6c62c 100644 --- a/python/upload_to_package_cloud.py +++ b/python/upload_to_package_cloud.py @@ -55,8 +55,6 @@ def upload_to_packagecloud(distro_name, package_name, packagecloud_token, repo_n def upload_files_in_directory_to_packagecloud(directoryName: str, distro_name: str, package_cloud_token: str, repo_name: str) -> MultipleReturnValue: - # print("Distro Name: " + distro_name) - # print("Supported Distros:") for key, value in supported_distros.items(): print(key + "=>" + str(value)) ret_status: List[ReturnValue] = [] @@ -104,7 +102,3 @@ def upload_files_in_directory_to_packagecloud(directoryName: str, distro_name: s sys.exit(0) else: sys.exit(1) -# for rv in multiple_return_value.return_values: -# if(rv.) -# print(r.message) -# print(r.success_status) From fb4199e2d89dbd2418775e37d3334b600fa3a832 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 23:47:56 +0300 Subject: [PATCH 045/106] Add new line at the end of some files Change test framework into pytest --- python/requirements.txt | 13 + python/tests/files/citus_include_10_0_3.spec | 3 + python/tests/files/pkgvars | 2 +- .../tests/test_update_package_properties.py | 335 +++++++++--------- python/update_package_properties.py | 2 +- 5 files changed, 194 insertions(+), 161 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index 0106dbd2..17ae4722 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -24,6 +24,7 @@ hyperlink==19.0.0 idna==2.8 importlib-metadata==1.5.0 incremental==16.10.1 +iniconfig==1.1.1 Jinja2==2.10.1 jsonpatch==1.22 jsonpointer==2.0 @@ -37,11 +38,18 @@ MarkupSafe==1.1.0 more-itertools==4.2.0 netifaces==0.10.4 oauthlib==3.1.0 +packaging==20.9 parameters-validation==1.2.0 pathlib2==2.3.5 +pbr==5.5.1 pexpect==4.6.0 +pipenv==2020.11.15 +pluggy==0.13.1 +py==1.10.0 pyasn1==0.4.2 pyasn1-modules==0.2.1 +pycairo==1.20.0 +pycurl==7.43.0.6 PyGithub==1.54.1 Pygments==2.3.1 PyGObject==3.36.0 @@ -50,8 +58,10 @@ PyJWT==1.7.1 pymacaroons==0.13.0 PyNaCl==1.3.0 pyOpenSSL==19.0.0 +pyparsing==2.4.7 pyrsistent==0.15.5 pyserial==3.4 +pytest==6.2.3 python-apt==2.0.0+ubuntu0.20.4.4 python-dateutil==2.7.3 python-debian===0.1.36ubuntu1 @@ -66,9 +76,12 @@ six==1.14.0 sos==4.0 ssh-import-id==5.10 systemd-python==234 +testresources==2.0.1 +toml==0.10.2 Twisted==18.9.0 ubuntu-advantage-tools==20.3 ufw==0.36 +unattended-upgrades==0.1 urllib3==1.25.8 virtualenv==20.0.17 wadllib==1.3.3 diff --git a/python/tests/files/citus_include_10_0_3.spec b/python/tests/files/citus_include_10_0_3.spec index 199fe73c..d378edc1 100644 --- a/python/tests/files/citus_include_10_0_3.spec +++ b/python/tests/files/citus_include_10_0_3.spec @@ -79,6 +79,9 @@ echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list %doc %{pginstdir}/doc/extension/README-%{sname}.md %changelog +* Thu Apr 4 2021 - Gurkan Indibay 10.0.3.citus-1 +- Official 10.0.3 release of Citus + * Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 - Official 10.0.2 release of Citus diff --git a/python/tests/files/pkgvars b/python/tests/files/pkgvars index df2ecae5..986e2100 100644 --- a/python/tests/files/pkgvars +++ b/python/tests/files/pkgvars @@ -3,4 +3,4 @@ pkgdesc='Citus (Open-Source)' pkglatest=10.0.2-1 releasepg=11,12,13 nightlypg=12,13 -versioning=fancy \ No newline at end of file +versioning=fancy diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index ae8ba62f..4f2853f2 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -1,5 +1,5 @@ +import pytest import os -import unittest from shutil import copyfile from .test_utils import are_strings_equal @@ -17,180 +17,197 @@ "CHANGELOG_DATE") is None else datetime.strptime(os.getenv("CHANGELOG_DATE"), '%a, %d %b %Y %H:%M:%S %z') -class PackagePropertiesTestCases(unittest.TestCase): +def test_get_version_number(): + assert get_version_number("10.0.3", True, 1) == "10.0.3-1" - def test_get_version_number(self): - self.assertEqual(get_version_number("10.0.3", True, 1), "10.0.3-1") - print(os.getenv("test")) - def test_get_version_number_with_project_name(self): - self.assertEqual(get_version_number_with_project_name("citus", "10.0.3", True, 1), "10.0.3.citus-1") +def test_get_version_number_with_project_name(): + assert get_version_number_with_project_name("citus", "10.0.3", True, 1) == "10.0.3.citus-1" - def test_find_nth_overlapping(self): - self.assertEqual(find_nth_overlapping("foofoo foofoo", "foofoo", 2), 7) - def test_get_changelog_for_tag(self): - changelog = get_changelog_for_tag(GITHUB_TOKEN, "citus", "v10.0.3") - with open(f"{TEST_BASE_PATH}/files/verify/expected_changelog_10.0.3.txt", "r") as reader: - expected_changelog = reader.read() - self.assertEqual(expected_changelog, changelog) +def test_find_nth_overlapping(): + assert find_nth_overlapping("foofoo foofoo", "foofoo", 2) == 7 - def test_get_debian_changelog_header(self): - header = get_debian_changelog_header("### citus v10.0.3 (March 16, 2021) ###", True, 2) - self.assertEqual(header, "citus (10.0.3.citus-2) stable; urgency=low") - def test_get_last_changelog_from_debian(self): - refer_file_path = f"{TEST_BASE_PATH}/files/verify/debian_changelog_with_10.0.3.txt" - expected_file_path = f"{TEST_BASE_PATH}/files/verify/expected_debian_latest_v10.0.3.txt" - with open(refer_file_path, "r") as reader: - changelog = reader.read() +def test_get_changelog_for_tag(): + changelog = get_changelog_for_tag(GITHUB_TOKEN, "citus", "v10.0.3") + with open(f"{TEST_BASE_PATH}/files/verify/expected_changelog_10.0.3.txt", "r") as reader: + expected_changelog = reader.read() + assert expected_changelog == changelog - latest_changelog = get_last_changelog_content_from_debian(changelog) - with open(expected_file_path, "r") as reader: - expected_output = reader.read() +def test_get_debian_changelog_header(): + header = get_debian_changelog_header("### citus v10.0.3 (March 16, 2021) ###", True, 2) + assert header == "citus (10.0.3.citus-2) stable; urgency=low" - are_strings_equal(expected_output, latest_changelog) - def test_prepend_latest_changelog_into_debian_changelog(self): - refer_file_path = f"{TEST_BASE_PATH}/files/debian.changelog.refer" - changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" - copyfile(refer_file_path, changelog_file_path) - latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) +def test_get_last_changelog_from_debian(): + refer_file_path = f"{TEST_BASE_PATH}/files/verify/debian_changelog_with_10.0.3.txt" + expected_file_path = f"{TEST_BASE_PATH}/files/verify/expected_debian_latest_v10.0.3.txt" + with open(refer_file_path, "r") as reader: + changelog = reader.read() - changelog_param = ChangelogParams() - changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( - MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( - 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog( - latest_changelog) - try: + latest_changelog = get_last_changelog_content_from_debian(changelog) + + with open(expected_file_path, "r") as reader: + expected_output = reader.read() + + are_strings_equal(expected_output, latest_changelog) + + +def test_prepend_latest_changelog_into_debian_changelog(): + refer_file_path = f"{TEST_BASE_PATH}/files/debian.changelog.refer" + changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" + copyfile(refer_file_path, changelog_file_path) + latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog( + latest_changelog) + try: + prepend_latest_changelog_into_debian_changelog(changelog_param, changelog_file_path) + verify_prepend_debian_changelog(changelog_file_path) + finally: + os.remove(changelog_file_path) + + +def test_prepend_latest_changelog_into_debian_changelog_10_0_3_already_included(): + refer_file_path = f"{TEST_BASE_PATH}/files/debian.changelog_include_10_0_3.refer" + changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" + copyfile(refer_file_path, changelog_file_path) + latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog( + latest_changelog) + try: + + with pytest.raises(ValueError): prepend_latest_changelog_into_debian_changelog(changelog_param, changelog_file_path) - self.verify_prepend_debian_changelog(changelog_file_path) - finally: - os.remove(changelog_file_path) - - def test_prepend_latest_changelog_into_debian_changelog_10_0_3_already_included(self): - refer_file_path = f"{TEST_BASE_PATH}/files/debian.changelog_include_10_0_3.refer" - changelog_file_path = f"{TEST_BASE_PATH}/files/debian.changelog" - copyfile(refer_file_path, changelog_file_path) - latest_changelog = get_changelog_for_tag(GITHUB_TOKEN, PROJECT_NAME, TAG_NAME) + finally: + os.remove(changelog_file_path) + + +def verify_prepend_debian_changelog(changelog_file_path): + with open(changelog_file_path, "r") as reader: + content = reader.read() + latest_changelog = get_last_changelog_content_from_debian(content) + with open(f"{TEST_BASE_PATH}/files/verify/expected_debian_latest_v10.0.3.txt", "r") as reader: + expected_content = reader.read() + are_strings_equal(expected_content, latest_changelog) + + +def test_convert_citus_changelog_into_rpm_changelog(): + changelog_param = ChangelogParams() + changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( + MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( + 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog("") + changelog = convert_citus_changelog_into_rpm_changelog(changelog_param) + with open(f"{TEST_BASE_PATH}/files/verify/rpm_latest_changelog_reference.txt", "r") as reader: + content = reader.read() + assert content == changelog + + +def test_update_rpm_spec(): + project_name = "citus" + spec_file = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" + spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" + spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" + templates_path = f"{BASE_PATH}/templates" + copyfile(spec_file, spec_file_copy) + try: changelog_param = ChangelogParams() changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( - 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog( - latest_changelog) - try: - prepend_latest_changelog_into_debian_changelog(changelog_param, changelog_file_path) - self.assertRaises(ValueError) - finally: - os.remove(changelog_file_path) - - def verify_prepend_debian_changelog(self, changelog_file_path): - with open(changelog_file_path, "r") as reader: - content = reader.read() - latest_changelog = get_last_changelog_content_from_debian(content) - with open(f"{TEST_BASE_PATH}/files/verify/expected_debian_latest_v10.0.3.txt", "r") as reader: - expected_content = reader.read() - are_strings_equal(expected_content, latest_changelog) - - def test_convert_citus_changelog_into_rpm_changelog(self): + 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") + update_rpm_spec(changelog_param, spec_file, templates_path) + verify_rpm_spec(spec_file_reference, spec_file) + finally: + copyfile(spec_file_copy, spec_file) + os.remove(spec_file_copy) + + +def test_update_rpm_spec_include_10_0_3(): + project_name = "citus" + spec_file = f"{TEST_BASE_PATH}/files/citus_include_10_0_3.spec" + spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" + templates_path = f"{BASE_PATH}/templates" + copyfile(spec_file, spec_file_copy) + try: changelog_param = ChangelogParams() changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( - 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog("") - changelog = convert_citus_changelog_into_rpm_changelog(changelog_param) - with open(f"{TEST_BASE_PATH}/files/verify/rpm_latest_changelog_reference.txt", "r") as reader: - content = reader.read() - self.assertEqual(content, changelog) - - def test_update_rpm_spec(self): - project_name = "citus" - spec_file = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" - spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" - spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" - templates_path = f"{BASE_PATH}/templates" - copyfile(spec_file, spec_file_copy) - try: - changelog_param = ChangelogParams() - changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( - MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( - 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") + 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") + with pytest.raises(ValueError): update_rpm_spec(changelog_param, spec_file, templates_path) - self.verify_rpm_spec(spec_file_reference, spec_file) - finally: - copyfile(spec_file_copy, spec_file) - os.remove(spec_file_copy) - - def test_update_rpm_spec_include_10_0_3(self): - project_name = "citus" - spec_file = f"{TEST_BASE_PATH}/files/citus_include_10_0_3.spec" - spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" - templates_path = f"{BASE_PATH}/templates" - copyfile(spec_file, spec_file_copy) - try: - changelog_param = ChangelogParams() - changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( - MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( - 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") - update_rpm_spec(changelog_param, spec_file, templates_path) - self.assertRaises(ValueError) - finally: - copyfile(spec_file_copy, spec_file) - os.remove(spec_file_copy) - - def verify_rpm_spec(self, spec_file_reference, spec_file_for_test): - with open(spec_file_for_test, "r") as reader_test: - with open(spec_file_reference, "r") as reader_reference: - test_str = reader_test.read() - reference_str = reader_reference.read() - are_strings_equal(reference_str, test_str) - - def test_update_pkg_vars(self): - templates_path = f"{BASE_PATH}/templates" - pkgvars_path = f"{TEST_BASE_PATH}/files/pkgvars" - pkgvars_copy_path = f"{pkgvars_path}_copy" - copyfile(pkgvars_path, pkgvars_copy_path) - try: - update_pkgvars("10.0.3", True, 1, templates_path, f"{TEST_BASE_PATH}/files/") - self.verify_pkgvars(pkgvars_path) - finally: - copyfile(pkgvars_copy_path, pkgvars_path) - os.remove(pkgvars_copy_path) - - def verify_pkgvars(self, pkgvars_path): - with open(pkgvars_path, "r") as reader: - content = reader.read() - index = content.find("pkglatest=10.0.3-1") - self.assertGreater(index, -1) - - def test_update_all_changes(self): - pkgvars_path = f"{TEST_BASE_PATH}/files/pkgvars" - pkgvars_copy_path = f"{pkgvars_path}_copy" - spec_file = f"{TEST_BASE_PATH}/files/{get_spec_file_name(PROJECT_NAME)}" - spec_file_copy = f"{spec_file}_copy" - spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(PROJECT_NAME)}" - - changelog_file_path = f"{TEST_BASE_PATH}/files/debian/changelog" - changelog_file_copy_path = f"{changelog_file_path}_copy" - copyfile(changelog_file_path, changelog_file_copy_path) - copyfile(pkgvars_path, pkgvars_copy_path) - copyfile(spec_file, spec_file_copy) - - try: - update_all_changes(GITHUB_TOKEN, PROJECT_NAME, PROJECT_VERSION, TAG_NAME, True, 1, MICROSOFT_EMAIL, - NAME_SURNAME, - CHANGELOG_DATE, f"{TEST_BASE_PATH}/files") - self.verify_prepend_debian_changelog(changelog_file_path) - self.verify_pkgvars(pkgvars_path) - self.verify_rpm_spec(spec_file_reference, spec_file) - finally: - copyfile(changelog_file_copy_path, changelog_file_path) - copyfile(pkgvars_copy_path, pkgvars_path) - copyfile(spec_file_copy, spec_file) - - os.remove(changelog_file_copy_path) - os.remove(pkgvars_copy_path) - os.remove(spec_file_copy) - - def test_regex(self): - print(re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", "### citus v10.0.3 (March 16, 2021) ###")) + finally: + copyfile(spec_file_copy, spec_file) + os.remove(spec_file_copy) + + +def verify_rpm_spec(spec_file_reference, spec_file_for_test): + with open(spec_file_for_test, "r") as reader_test: + with open(spec_file_reference, "r") as reader_reference: + test_str = reader_test.read() + reference_str = reader_reference.read() + are_strings_equal(reference_str, test_str) + + +def test_update_pkg_vars(): + templates_path = f"{BASE_PATH}/templates" + pkgvars_path = f"{TEST_BASE_PATH}/files/pkgvars" + pkgvars_copy_path = f"{pkgvars_path}_copy" + copyfile(pkgvars_path, pkgvars_copy_path) + try: + update_pkgvars("10.0.3", True, 1, templates_path, f"{TEST_BASE_PATH}/files/") + verify_pkgvars(pkgvars_path) + finally: + copyfile(pkgvars_copy_path, pkgvars_path) + os.remove(pkgvars_copy_path) + + +def verify_pkgvars(pkgvars_path): + with open(pkgvars_path, "r") as reader: + content = reader.read() + index = content.find("pkglatest=10.0.3-1") + assert index > -1 + + +def test_update_all_changes(): + pkgvars_path = f"{TEST_BASE_PATH}/files/pkgvars" + pkgvars_copy_path = f"{pkgvars_path}_copy" + spec_file = f"{TEST_BASE_PATH}/files/{get_spec_file_name(PROJECT_NAME)}" + spec_file_copy = f"{spec_file}_copy" + spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(PROJECT_NAME)}" + + changelog_file_path = f"{TEST_BASE_PATH}/files/debian/changelog" + changelog_file_copy_path = f"{changelog_file_path}_copy" + copyfile(changelog_file_path, changelog_file_copy_path) + copyfile(pkgvars_path, pkgvars_copy_path) + copyfile(spec_file, spec_file_copy) + + try: + update_all_changes(GITHUB_TOKEN, PROJECT_NAME, PROJECT_VERSION, TAG_NAME, True, 1, MICROSOFT_EMAIL, + NAME_SURNAME, + CHANGELOG_DATE, f"{TEST_BASE_PATH}/files") + verify_prepend_debian_changelog(changelog_file_path) + verify_pkgvars(pkgvars_path) + verify_rpm_spec(spec_file_reference, spec_file) + finally: + copyfile(changelog_file_copy_path, changelog_file_path) + copyfile(pkgvars_copy_path, pkgvars_path) + copyfile(spec_file_copy, spec_file) + + os.remove(changelog_file_copy_path) + os.remove(pkgvars_copy_path) + os.remove(spec_file_copy) + + +def test_regex(): + print(re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", "### citus v10.0.3 (March 16, 2021) ###")) + + + diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 86806f63..0ea1b89f 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -231,7 +231,7 @@ def prepend_latest_changelog_into_debian_changelog(changelog_param: ChangelogPar reader.seek(0, 0) reader.write(changelog) else: - print("Already version in the debian changelog") + raise ValueError("Already version in the debian changelog") @validate_parameters From 3855dd253c7619a7f9e1dff06a698f4015f317ff Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 23:51:44 +0300 Subject: [PATCH 046/106] Add missing dependencies for pycurl --- .github/workflows/tool-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 88a47b8e..66a45c8f 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -15,7 +15,8 @@ jobs: steps: - uses: actions/checkout@v2 - + - name: intall dependencies + run: sudo apt install libcurl4-openssl-dev libssl-dev - name: install requirements run: python -m pip install -r python/requirements.txt - name: Execute Unit tests From 0b47a81ba6ac7d095412ffb66eebe8a072b50061 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 23:54:02 +0300 Subject: [PATCH 047/106] Remove unnecessary package unattended-upgrades --- python/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index 17ae4722..e746e2a2 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -81,7 +81,6 @@ toml==0.10.2 Twisted==18.9.0 ubuntu-advantage-tools==20.3 ufw==0.36 -unattended-upgrades==0.1 urllib3==1.25.8 virtualenv==20.0.17 wadllib==1.3.3 From ef500824048bfdc8ec93ae9b707c1fa4fccf1b78 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 23:55:58 +0300 Subject: [PATCH 048/106] Add pytest execution command on pipeline file --- .github/workflows/tool-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 66a45c8f..78cd5e28 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -15,9 +15,9 @@ jobs: steps: - uses: actions/checkout@v2 - - name: intall dependencies + - name: install dependencies run: sudo apt install libcurl4-openssl-dev libssl-dev - name: install requirements run: python -m pip install -r python/requirements.txt - name: Execute Unit tests - run: python -m unittest python/tests/test_update_package_properties.py + run: python -m pytest -q python/tests/test_update_package_properties.py From 46c07f2b72f92a544083edf9f76e11fbf074e62a Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 12 Apr 2021 23:57:58 +0300 Subject: [PATCH 049/106] Add __init__.py files for module definitions --- python/__init__.py | 0 python/tests/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/__init__.py create mode 100644 python/tests/__init__.py diff --git a/python/__init__.py b/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 9bff43caa2c117c313cc00b787c863dd19f1ff8a Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 10:14:50 +0300 Subject: [PATCH 050/106] Add validation controls into ChangelogParams setters Remove unused code blocks --- python/update_package_properties.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 0ea1b89f..068ad46a 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -66,14 +66,14 @@ def set_latest_changelog(self, param: str): def get_project_name(self) -> str: return self.__project_name - def set_project_name(self, param: str): + def set_project_name(self, param: non_empty(non_blank(str))): self.__project_name = param return self def get_project_version(self) -> str: return self.__project_version - def set_project_version(self, param: str): + def set_project_version(self, param: is_version(str)): self.__project_version = param return self @@ -87,21 +87,21 @@ def set_fancy(self, param: bool): def get_fancy_version_number(self) -> int: return self.__fancy_version_number - def set_fancy_version_number(self, param: int): + def set_fancy_version_number(self, param: non_negative(int)): self.__fancy_version_number = param return self def get_microsoft_email(self) -> str: return self.__microsoft_email - def set_microsoft_email(self, param: str): + def set_microsoft_email(self, param: is_email(str)): self.__microsoft_email = param return self def get_name_surname(self) -> str: return self.__name_surname - def set_name_surname(self, param: str): + def set_name_surname(self, param: non_empty(non_blank(str))): self.__name_surname = param return self @@ -268,20 +268,6 @@ def get_debian_trailer(microsoft_email: str, name_surname: str, changelog_date: return f" -- {name_surname} <{microsoft_email}> {formatted_date} \n " -def get_rpm_changelog_from_actual_changelog(project_name: str, project_version: str, latest_changelog: str, - microsoft_email: str, - name_surname: str, fancy: bool, fancy_version_number: int, - spec_file_path: str): - rpm_changelog_history = get_rpm_changelog_history(spec_file_path) - changelog_histories = rpm_changelog_history.splitlines() - if len(changelog_histories) > 1 and project_version in changelog_histories[1]: - return rpm_changelog_history - - header = get_rpm_header(project_name, microsoft_email, name_surname, project_version, fancy, - fancy_version_number, date.today()) - return '\n'.join([header, latest_changelog, rpm_changelog_history]) + "\n" - - def convert_citus_changelog_into_rpm_changelog(changelog_params: ChangelogParams) -> str: header = get_rpm_header(changelog_params) rpm_changelog = f"{header.strip()}\n- Update to {changelog_params.get_project_name().capitalize()} {changelog_params.get_project_version()}" From 872ff5cd5f86cf40224e7a7b2795305e187b60fe Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 10:18:50 +0300 Subject: [PATCH 051/106] Remove unused imports Fix some formatting issues --- python/update_package_properties.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 913b3174..af1b7107 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -1,8 +1,7 @@ import argparse -import re -from datetime import date, datetime +from datetime import date + import pathlib2 -import string_utils from github import Github from parameters_validation import no_whitespaces, non_blank, non_empty, non_negative, validate_parameters @@ -10,6 +9,7 @@ BASE_PATH = pathlib2.Path(__file__).parent.absolute() + class ChangelogParams: __latest_changelog: str = "" __project_version: str = "" From 22fc2139cd8d9561c242ec9884a3edfeb7e64841 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 10:21:52 +0300 Subject: [PATCH 052/106] Change the release_date datatype into datetime to match the internal data type --- python/update_package_properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 068ad46a..af486361 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -302,7 +302,7 @@ def update_all_changes(github_token: non_empty(non_blank(str)), project_name: no project_version: is_version(str), tag_name: is_tag(str), fancy: bool, fancy_version_number: non_negative(int), microsoft_email: is_email(str), - name_surname: non_empty(non_blank(str)), release_date: date, packaging_path: str): + name_surname: non_empty(non_blank(str)), release_date: datetime, packaging_path: str): templates_path = f"{BASE_PATH}/templates" update_pkgvars(project_version, fancy, fancy_version_number, templates_path, f"{packaging_path}") latest_changelog = get_changelog_for_tag(github_token, project_name, tag_name) From 39606badb9560f2e3a41f9a703223989e11c39fa Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 12:02:10 +0300 Subject: [PATCH 053/106] Move test_prepare_release into pytest --- .github/workflows/tool-tests.yml | 2 +- python/tests/test_prepare_release.py | 213 +++++++++++++-------------- 2 files changed, 104 insertions(+), 111 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 4ce0f20e..d7e73277 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -28,7 +28,7 @@ jobs: run: python -m pytest -q python/tests/test_update_package_properties.py - name: Unit tests for "Prepare Release" - run: python -m unittest python/tests/test_prepare_release.py + run: python -m pytest -q python/tests/test_prepare_release.py diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index 1e66baa3..f0cc61f2 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -1,5 +1,3 @@ -import unittest - import pathlib2 from ..prepare_release import * @@ -10,116 +8,111 @@ TEST_BASE_PATH = f"{BASE_PATH}/citus" CITUS_PROJECT_TEST_PATH = f"{TEST_BASE_PATH}/projects/citus" CITUS_PROJECT_TEST_PATH_COPY = f"{TEST_BASE_PATH}/projects/citus_copy" - - -class PrepareReleaseTestCases(unittest.TestCase): - def setUp(self): - if os.path.exists("citus"): - run("rm -r citus") - self.initialize_env() - - def initialize_env(self): - # print("Current Directory:"+os.getcwd()) - if not os.path.exists("citus"): - run("git clone https://github.com/citusdata/citus.git") - - def test_major_release(self): - self.initialize_env() - os.chdir("citus") - release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( - github_token=github_token, - project_name="citus", - project_version="10.2.0", - main_branch="master", - earliest_pr_date=datetime.strptime( - '2021.03.25 00:00', - '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, - is_test=True) - # run(f"git checkout {release_branch}") - - # newly_created_sql_file = f"{DISTRIBUTED_DIR_PATH}/citus--10.1-1--10.2-1.sql" - - run(f"git checkout {release_branch}") - - self.assertTrue(has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.0")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.0])")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.0'")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, - "\`configure' configures Citus 10.2.0 to adapt to many kinds of systems.")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.0:";;')) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'")) - - run(f"git checkout {upcoming_version_branch}") - - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'")) - self.assertTrue( - has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, "ALTER EXTENSION citus UPDATE TO '10.2-1';")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel")) - self.assertEqual(2, line_count_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, - "ALTER EXTENSION citus UPDATE TO '10.2-1';")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, - "\`configure' configures Citus 10.2devel to adapt to many kinds of systems.")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;')) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'")) - self.assertTrue(os.path.exists(f"{TEST_BASE_PATH}/{newly_created_sql_file}")) - # - - if resource_status in (ResourceStatus.RELEASE_BRANCH_LOCAL, ResourceStatus.RELEASE_BRANCH_REMOTE, - ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, - ResourceStatus.PULL_REQUEST_CREATED): - run(f"git branch -D {release_branch} ") - elif resource_status in (ResourceStatus.RELEASE_BRANCH_REMOTE, - ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, - ResourceStatus.PULL_REQUEST_CREATED): - run(f"git push origin --delete {release_branch}") - elif resource_status in (ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, - ResourceStatus.PULL_REQUEST_CREATED): - run(f"git branch -D {upcoming_version_branch} ") - elif resource_status in (ResourceStatus.UPCOMING_VERSION_REMOTE, - ResourceStatus.PULL_REQUEST_CREATED): - run(f"git push origin --delete {release_branch}") - self.clear_env() - - def test_patch_release(self): - self.initialize_env() - # print(os.getcwd()) - os.chdir("citus") - - update_release(github_token=github_token, project_name="citus", project_version="10.2.0", - main_branch="master", - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True) - - update_release(github_token=github_token, project_name="citus", project_version="10.2.1", - main_branch="master", - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, - "\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.")) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.1:";;')) - self.assertTrue(has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'")) - - self.clear_env() +MAIN_BRANCH = "test-tools-scripts" + + +def initialize_env(): + if not os.path.exists("citus"): + run("git clone https://github.com/citusdata/citus.git") + + +def test_major_release(): + initialize_env() + os.chdir("citus") + + release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( + github_token=github_token, + project_name="citus", + project_version="10.2.0", + main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime( + '2021.03.25 00:00', + '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, + is_test=True) + + run(f"git checkout {release_branch}") + + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.0") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.0])") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.0'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2.0 to adapt to many kinds of systems.") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2.0:";;') + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") + + run(f"git checkout {upcoming_version_branch}") + + assert has_file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert line_count_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 + assert has_file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert os.path.exists(f"{TEST_BASE_PATH}/{newly_created_sql_file}") + # + run(f"git checkout {MAIN_BRANCH}") + clear_branches(release_branch, resource_status, upcoming_version_branch) + clear_env() + + +def clear_branches(release_branch, resource_status, upcoming_version_branch): + if resource_status in (ResourceStatus.RELEASE_BRANCH_LOCAL, ResourceStatus.RELEASE_BRANCH_REMOTE, + ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, + ResourceStatus.PULL_REQUEST_CREATED): + run(f"git branch -D {release_branch} ") + + if resource_status in (ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, + ResourceStatus.PULL_REQUEST_CREATED): + run(f"git branch -D {upcoming_version_branch} ") + + +def test_patch_release(): + initialize_env() + os.chdir("citus") + + update_release(github_token=github_token, project_name="citus", project_version="10.2.0", + main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, is_test=True) + + release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( + github_token=github_token, project_name="citus", project_version="10.2.1", + main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, is_test=True) + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + run(f"git checkout {MAIN_BRANCH}") + + clear_branches(release_branch, resource_status, upcoming_version_branch) + + clear_env() # def tearDown(self): # self.clear_env() - def clear_env(self): - if os.path.exists("../citus"): - os.chdir("..") - run("chmod -R 777 citus") - run("sudo rm -rf citus") +def clear_env(): + if os.path.exists("../citus"): + os.chdir("..") + run("chmod -R 777 citus") + run("sudo rm -rf citus") From e55710c1117f3a0b47ecae2a04ab6656aa1e2b4e Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 13:37:07 +0300 Subject: [PATCH 054/106] Fix spec file rpm version problem which causes build fail --- python/templates/project.spec.tmpl | 2 +- python/tests/files/citus.spec | 4 ++-- python/tests/files/citus_include_10_0_3.spec | 4 ++-- .../files/verify/rpm_latest_changelog_reference.txt | 2 +- python/tests/test_update_package_properties.py | 6 +++--- python/update_package_properties.py | 11 +++++++---- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/python/templates/project.spec.tmpl b/python/templates/project.spec.tmpl index ae0ace57..ebff4967 100644 --- a/python/templates/project.spec.tmpl +++ b/python/templates/project.spec.tmpl @@ -7,7 +7,7 @@ Summary: PostgreSQL-based distributed RDBMS Name: %{sname}%{?pkginfix}_%{pgmajorversion} Provides: %{sname}_%{pgmajorversion} Conflicts: %{sname}_%{pgmajorversion} -Version: {{fancy_version_str}} +Version: {{rpm_version}} Release: {{fancy_version_no}}%{dist} License: AGPLv3 Group: Applications/Databases diff --git a/python/tests/files/citus.spec b/python/tests/files/citus.spec index 199fe73c..753a9946 100644 --- a/python/tests/files/citus.spec +++ b/python/tests/files/citus.spec @@ -7,11 +7,11 @@ Summary: PostgreSQL-based distributed RDBMS Name: %{sname}%{?pkginfix}_%{pgmajorversion} Provides: %{sname}_%{pgmajorversion} Conflicts: %{sname}_%{pgmajorversion} -Version: 10.0.3-1 +Version: 10.0.2.citus Release: 1%{dist} License: AGPLv3 Group: Applications/Databases -Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz +Source0: https://github.com/citusdata//archive/v10.0.2.tar.gz URL: https://github.com/citusdata/ BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel Requires: postgresql%{pgmajorversion}-server diff --git a/python/tests/files/citus_include_10_0_3.spec b/python/tests/files/citus_include_10_0_3.spec index d378edc1..716995c0 100644 --- a/python/tests/files/citus_include_10_0_3.spec +++ b/python/tests/files/citus_include_10_0_3.spec @@ -7,7 +7,7 @@ Summary: PostgreSQL-based distributed RDBMS Name: %{sname}%{?pkginfix}_%{pgmajorversion} Provides: %{sname}_%{pgmajorversion} Conflicts: %{sname}_%{pgmajorversion} -Version: 10.0.3-1 +Version: 10.0.3.citus Release: 1%{dist} License: AGPLv3 Group: Applications/Databases @@ -79,7 +79,7 @@ echo %{pginstdir}/share/extension/%{sname}.control >> installation_files.list %doc %{pginstdir}/doc/extension/README-%{sname}.md %changelog -* Thu Apr 4 2021 - Gurkan Indibay 10.0.3.citus-1 +* Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 - Official 10.0.3 release of Citus * Thu Mar 4 2021 - Gurkan Indibay 10.0.2.citus-1 diff --git a/python/tests/files/verify/rpm_latest_changelog_reference.txt b/python/tests/files/verify/rpm_latest_changelog_reference.txt index 3a2f0911..6518d086 100644 --- a/python/tests/files/verify/rpm_latest_changelog_reference.txt +++ b/python/tests/files/verify/rpm_latest_changelog_reference.txt @@ -1,2 +1,2 @@ * Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 -- Update to Citus 10.0.3 \ No newline at end of file +- Official 10.0.3 release of Citus \ No newline at end of file diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index 4f2853f2..2a3846f8 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -115,14 +115,14 @@ def test_update_rpm_spec(): project_name = "citus" spec_file = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" spec_file_copy = f"{os.getcwd()}/{get_spec_file_name(project_name)}_copy" - spec_file_reference = f"{TEST_BASE_PATH}/files/{get_spec_file_name(project_name)}" + spec_file_reference = f"{TEST_BASE_PATH}/files/citus_include_10_0_3.spec" templates_path = f"{BASE_PATH}/templates" copyfile(spec_file, spec_file_copy) try: changelog_param = ChangelogParams() changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( - 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") + 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog("") update_rpm_spec(changelog_param, spec_file, templates_path) verify_rpm_spec(spec_file_reference, spec_file) finally: @@ -140,7 +140,7 @@ def test_update_rpm_spec_include_10_0_3(): changelog_param = ChangelogParams() changelog_param.set_project_version(PROJECT_VERSION).set_project_name(PROJECT_NAME).set_microsoft_email( MICROSOFT_EMAIL).set_name_surname(NAME_SURNAME).set_fancy_version_number( - 1).set_changelog_date(datetime.now()).set_fancy(True).set_latest_changelog("") + 1).set_changelog_date(CHANGELOG_DATE).set_fancy(True).set_latest_changelog("") with pytest.raises(ValueError): update_rpm_spec(changelog_param, spec_file, templates_path) finally: diff --git a/python/update_package_properties.py b/python/update_package_properties.py index af486361..3c1944cb 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -122,6 +122,10 @@ def get_version_number(version: str, fancy: bool, fancy_release_count: int) -> s return f"{version}{fancy_suffix}" +def get_rpm_version(project_name: str, version: str) -> str: + return f"{version}.{project_name}" + + def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, fancy_release_count: int) -> str: fancy_suffix = f"-{fancy_release_count}" if fancy else "" @@ -270,7 +274,7 @@ def get_debian_trailer(microsoft_email: str, name_surname: str, changelog_date: def convert_citus_changelog_into_rpm_changelog(changelog_params: ChangelogParams) -> str: header = get_rpm_header(changelog_params) - rpm_changelog = f"{header.strip()}\n- Update to {changelog_params.get_project_name().capitalize()} {changelog_params.get_project_version()}" + rpm_changelog = f"{header.strip()}\n- Official {changelog_params.get_project_version()} release of {changelog_params.get_project_name().capitalize()} " return rpm_changelog @@ -279,8 +283,7 @@ def update_rpm_spec(changelog_param: ChangelogParams, spec_file_name: str, templates_path: str) -> None: env = get_template_environment(templates_path) - fancy_version_str = get_version_number(changelog_param.get_project_version(), changelog_param.get_fancy(), - changelog_param.get_fancy_version_number()) + rpm_version = get_rpm_version(changelog_param.get_project_name(), changelog_param.get_project_version()) template = env.get_template('project.spec.tmpl') rpm_changelog_history = get_rpm_changelog_history(spec_file_name) @@ -291,7 +294,7 @@ def update_rpm_spec(changelog_param: ChangelogParams, spec_file_name: str, latest_changelog = convert_citus_changelog_into_rpm_changelog(changelog_param) changelog = f"{latest_changelog}\n\n{rpm_changelog_history}" - content = template.render(version=changelog_param.get_project_version(), fancy_version_str=fancy_version_str, + content = template.render(version=changelog_param.get_project_version(), rpm_version=rpm_version, fancy_version_no=changelog_param.get_fancy_version_number(), changelog=changelog) with open(spec_file_name, "w+") as writer: writer.write(content) From 8698ee1678320392b8c8249f3b8efff85d38322a Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 13:48:52 +0300 Subject: [PATCH 055/106] Fix missing project name problem on rpm spec --- python/tests/files/citus_include_10_0_3.spec | 4 ++-- python/update_package_properties.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/tests/files/citus_include_10_0_3.spec b/python/tests/files/citus_include_10_0_3.spec index 716995c0..d1c25c25 100644 --- a/python/tests/files/citus_include_10_0_3.spec +++ b/python/tests/files/citus_include_10_0_3.spec @@ -11,8 +11,8 @@ Version: 10.0.3.citus Release: 1%{dist} License: AGPLv3 Group: Applications/Databases -Source0: https://github.com/citusdata//archive/v10.0.3.tar.gz -URL: https://github.com/citusdata/ +Source0: https://github.com/citusdata/citus/archive/v10.0.3.tar.gz +URL: https://github.com/citusdata/citus BuildRequires: postgresql%{pgmajorversion}-devel libcurl-devel Requires: postgresql%{pgmajorversion}-server BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 3c1944cb..b8fee8aa 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -295,6 +295,7 @@ def update_rpm_spec(changelog_param: ChangelogParams, spec_file_name: str, latest_changelog = convert_citus_changelog_into_rpm_changelog(changelog_param) changelog = f"{latest_changelog}\n\n{rpm_changelog_history}" content = template.render(version=changelog_param.get_project_version(), rpm_version=rpm_version, + project_name=changelog_param.get_project_name(), fancy_version_no=changelog_param.get_fancy_version_number(), changelog=changelog) with open(spec_file_name, "w+") as writer: writer.write(content) From 1bb9beae04148cb1996dceb91b1a599dc82b54cf Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 13:54:15 +0300 Subject: [PATCH 056/106] Remove empty character that cause tests fail from rpm_changelog converter --- python/update_package_properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/update_package_properties.py b/python/update_package_properties.py index b8fee8aa..a72bf57a 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -274,7 +274,7 @@ def get_debian_trailer(microsoft_email: str, name_surname: str, changelog_date: def convert_citus_changelog_into_rpm_changelog(changelog_params: ChangelogParams) -> str: header = get_rpm_header(changelog_params) - rpm_changelog = f"{header.strip()}\n- Official {changelog_params.get_project_version()} release of {changelog_params.get_project_name().capitalize()} " + rpm_changelog = f"{header.strip()}\n- Official {changelog_params.get_project_version()} release of {changelog_params.get_project_name().capitalize()}" return rpm_changelog From b4d17612b239715c979166d6e60744125325a843 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 13 Apr 2021 14:11:23 +0300 Subject: [PATCH 057/106] Fix pkgvars version style --- python/tests/test_update_package_properties.py | 7 ++----- python/update_package_properties.py | 8 ++++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index 2a3846f8..240ac95c 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -162,7 +162,7 @@ def test_update_pkg_vars(): pkgvars_copy_path = f"{pkgvars_path}_copy" copyfile(pkgvars_path, pkgvars_copy_path) try: - update_pkgvars("10.0.3", True, 1, templates_path, f"{TEST_BASE_PATH}/files/") + update_pkgvars(PROJECT_NAME, PROJECT_VERSION, True, 1, templates_path, f"{TEST_BASE_PATH}/files/") verify_pkgvars(pkgvars_path) finally: copyfile(pkgvars_copy_path, pkgvars_path) @@ -172,7 +172,7 @@ def test_update_pkg_vars(): def verify_pkgvars(pkgvars_path): with open(pkgvars_path, "r") as reader: content = reader.read() - index = content.find("pkglatest=10.0.3-1") + index = content.find(f"pkglatest={PROJECT_VERSION}.{PROJECT_NAME}-1") assert index > -1 @@ -208,6 +208,3 @@ def test_update_all_changes(): def test_regex(): print(re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", "### citus v10.0.3 (March 16, 2021) ###")) - - - diff --git a/python/update_package_properties.py b/python/update_package_properties.py index a72bf57a..dfdb032c 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -239,15 +239,15 @@ def prepend_latest_changelog_into_debian_changelog(changelog_param: ChangelogPar @validate_parameters -def update_pkgvars(version: is_version(non_empty(no_whitespaces(non_blank(str)))), fancy: bool, +def update_pkgvars(project_name: str, version: is_version(non_empty(no_whitespaces(non_blank(str)))), fancy: bool, fancy_release_count: non_negative(int), templates_path: str, pkgvars_path: str) -> None: env = get_template_environment(templates_path) - version_str = get_version_number(version, fancy, fancy_release_count) + version_str = get_version_number_with_project_name(project_name,version, fancy, fancy_release_count) template = env.get_template('pkgvars.tmpl') - pkgvars_content = template.render(version=version_str) + pkgvars_content = f"{template.render(version=version_str)}\n" with open(f'{pkgvars_path}/pkgvars', "w") as writer: writer.write(pkgvars_content) @@ -308,7 +308,7 @@ def update_all_changes(github_token: non_empty(non_blank(str)), project_name: no microsoft_email: is_email(str), name_surname: non_empty(non_blank(str)), release_date: datetime, packaging_path: str): templates_path = f"{BASE_PATH}/templates" - update_pkgvars(project_version, fancy, fancy_version_number, templates_path, f"{packaging_path}") + update_pkgvars(project_name, project_version, fancy, fancy_version_number, templates_path, f"{packaging_path}") latest_changelog = get_changelog_for_tag(github_token, project_name, tag_name) changelog_param = ChangelogParams() changelog_param.set_project_version(project_version).set_project_name(project_name).set_microsoft_email( From c13b72cac5a133c208687bc47ab1874765e72f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 3 May 2021 17:03:35 +0300 Subject: [PATCH 058/106] Simplify method find_nth_overlapping_line_by_regex Method simplified with the suggestion of @onur Co-authored-by: Onur Tirtir --- python/common_tool_methods.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/common_tool_methods.py b/python/common_tool_methods.py index 1a831119..720db8aa 100644 --- a/python/common_tool_methods.py +++ b/python/common_tool_methods.py @@ -41,14 +41,12 @@ def find_nth_overlapping(subject_string, search_string, n) -> int: def find_nth_overlapping_line_by_regex(subject_string, regex_pattern, n) -> int: lines = subject_string.splitlines() counter = 0 - index = -1 for i in range(len(lines)): if re.match(regex_pattern, lines[i]): counter = counter + 1 if counter == n: - index = i - break - return index + return i + return -1 def remove_string_inside_parentheses(param: str) -> str: From a2e8aef98e41c785588ddb62c2ed0f902bee180d Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 4 May 2021 16:16:50 +0300 Subject: [PATCH 059/106] Fix pr #91 comments --- python/common_tool_methods.py | 47 ++-- python/common_validations.py | 15 +- python/prepare_release.py | 327 +++++++++++++---------- python/tests/test_common_tool_methods.py | 2 +- python/tests/test_prepare_release.py | 146 +++++----- 5 files changed, 282 insertions(+), 255 deletions(-) diff --git a/python/common_tool_methods.py b/python/common_tool_methods.py index 720db8aa..16a3ada6 100644 --- a/python/common_tool_methods.py +++ b/python/common_tool_methods.py @@ -2,6 +2,7 @@ import subprocess from datetime import datetime from typing import Dict, List +import os from github import Repository, PullRequest from jinja2 import Environment, FileSystemLoader @@ -71,16 +72,24 @@ def get_version_details(version: is_version(str)) -> Dict[str, str]: return {"major": version_parts[0], "minor": version_parts[1], "patch": version_parts[2]} +def get_default_upcoming_version(version: is_version(str)) -> str: + project_version_details = get_version_details(version) + return f'{project_version_details["major"]}.{project_version_details["minor"]}.' \ + f'{str(int(project_version_details["patch"]) + 1)}' + + +def get_upcoming_minor_version(version: is_version(str)) -> str: + upcoming_version_details = get_version_details(version) + return f'{upcoming_version_details["major"]}.{upcoming_version_details["minor"]}' + + def is_major_release(version: is_version(str)) -> bool: version_info = get_version_details(version) return version_info["patch"] == "0" def str_array_to_str(str_array: List[str]) -> str: - result_str = "" - for i in range(len(str_array)): - result_str = result_str + str_array[i] + "\n" - return result_str + return f"{os.linesep.join(str_array)}{os.linesep}" def get_prs(repo: Repository.Repository, earliest_date: datetime, base_branch: str, last_date: datetime = None): @@ -89,20 +98,17 @@ def get_prs(repo: Repository.Repository, earliest_date: datetime, base_branch: s for pull in pulls: if not pull.is_merged(): continue - if last_date is not None and pull.created_at > last_date: - continue if pull.created_at < earliest_date: break - if pull.merged_at is not None and pull.merged_at > earliest_date: + if pull.merged_at is not None and earliest_date < pull.merged_at < last_date: picked_pulls.append(pull) return picked_pulls -def get_pr_issues_by_label(prs: List[PullRequest.PullRequest], label_name: str): +def get_prs_by_label(prs: List[PullRequest.PullRequest], label_name: str): filtered_prs = [] for pr in prs: - filtered_labels = filter(lambda label: label.name == label_name, pr.labels) - if len(list(filtered_labels)) > 0: + if any(label.name == label_name for label in pr.labels): filtered_prs.append(pr) return filtered_prs @@ -111,37 +117,28 @@ def has_file_include_line(base_path: str, relative_file_path: str, line_content: with open(f"{base_path}/{relative_file_path}", "r") as reader: content = reader.read() lines = content.splitlines() - found = False for line in lines: if line == line_content: - found = True - break - return found + return True + return False -def line_count_in_file(base_path: str, relative_file_path: str, line_content: str) -> int: +def line_count_in_file(base_path: str, relative_file_path: str, search_line: str) -> int: with open(f"{base_path}/{relative_file_path}", "r") as reader: content = reader.read() lines = content.splitlines() - counter = 0 - for line in lines: - if line == line_content: - counter = counter + 1 - - return counter + return len(list(filter(lambda line: line == search_line, lines))) def replace_line_in_file(file: str, match_regex: str, replace_str: str) -> bool: with open(file, "r") as reader: file_content = reader.read() lines = file_content.splitlines() - line_counter = 0 has_match = False - for line in lines: + for line_number, line in enumerate(lines): if re.match(match_regex, line): has_match = True - lines[line_counter] = replace_str - line_counter = line_counter + 1 + lines[line_number] = replace_str edited_content = str_array_to_str(lines) with open(file, "w") as writer: writer.write(edited_content) diff --git a/python/common_validations.py b/python/common_validations.py index 9b47805c..9a3f181d 100644 --- a/python/common_validations.py +++ b/python/common_validations.py @@ -2,24 +2,25 @@ from parameters_validation import parameter_validation import re +CITUS_VERSION_PATTERN = r"\d{1,2}\.\d{1,2}\.\d{1,2}$" + @parameter_validation def is_version(version: str): - if version is None or not version: + if not version: raise ValueError("version should be non-empty and should not be None") - if not re.match(r"\d+\.\d+\.\d+$", version): + if not re.match(CITUS_VERSION_PATTERN, version): raise ValueError( - "version should include 3 levels of versions consists of numbers separated with dots. e.g: 10.0.1") + "version should include three level of digits separated by dots, e.g: 10.0.1") @parameter_validation def is_tag(tag: str): - if tag is None or not tag: + if not tag: raise ValueError("tag should be non-empty and should not be None") - if not re.match(r"v\d+\.\d+\.\d+$", tag): + if not re.match(f"v{CITUS_VERSION_PATTERN}", tag): raise ValueError( - "tag should start with v and should include 3 levels of versions consists of numbers " + - "separated with dots. e.g: v10.0.1") + "tag should start with 'v' and should include three level of digits separated by dots, e.g: v10.0.1") @parameter_validation diff --git a/python/prepare_release.py b/python/prepare_release.py index 9ea9d9b8..76617bae 100644 --- a/python/prepare_release.py +++ b/python/prepare_release.py @@ -1,11 +1,11 @@ -import os import uuid +from enum import Enum from typing import Tuple from github import Github from .common_tool_methods import * -from enum import Enum +from .common_validations import * MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" CITUS_CONTROL = "src/backend/distributed/citus.control" @@ -16,8 +16,8 @@ CONFIGURE = "configure" CITUS_CONTROL_SEARCH_PATTERN = r"^default_version*" -MULT_EXT_MAJOR_SEARCH_PATTERN = r"^\s*\d{1,2}\.\d{1,2}devel$" -MULT_EXT_PATCH_SEARCH_PATTERN = r"^\s*\d{1,2}\.\d{1,2}.\d{1,2}$" +MULTI_EXT_MAJOR_SEARCH_PATTERN = r"^\s*\d{1,2}\.\d{1,2}devel$" +MULTI_EXT_MINOR_PATCH_SEARCH_PATTERN = r"^\s*" + CITUS_VERSION_PATTERN CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" REPO_OWNER = "citusdata" @@ -32,6 +32,11 @@ class ResourceStatus(Enum): PULL_REQUEST_CREATED = 6 +def get_minor_version(version: str) -> str: + project_version_details = get_version_details(version) + return f'{project_version_details["major"]}.{project_version_details["minor"]}' + + def update_release(github_token: str, project_name: str, project_version: is_version(str), main_branch: str, earliest_pr_date: datetime, exec_path: str, is_test: bool = False, cherry_pick_enabled: bool = False) -> Tuple[str, str, str, ResourceStatus]: @@ -45,212 +50,238 @@ def update_release(github_token: str, project_name: str, project_version: is_ver resource_status = ResourceStatus.INITIAL project_version_details = get_version_details(project_version) - default_upcoming_version = f'{project_version_details["major"]}.{project_version_details["minor"]}.' \ - f'{str(int(project_version_details["patch"]) + 1)}' - upcoming_version = default_upcoming_version if os.getenv("UPCOMING_VERSION") is None else os.getenv( - "UPCOMING_VERSION") - upcoming_version_details = get_version_details(upcoming_version) - upcoming_patch_version = f'{upcoming_version_details["major"]}.{upcoming_version_details["minor"]}' - devel_version = f"{upcoming_patch_version}devel" + default_upcoming_version = get_default_upcoming_version(project_version) + upcoming_version = os.getenv("UPCOMING_VERSION", default=default_upcoming_version) + upcoming_minor_version = get_minor_version(upcoming_version) + devel_version = f"{upcoming_minor_version}devel" + release_branch_name = f'release-{project_version_details["major"]}.{project_version_details["minor"]}' release_branch_name = f"{release_branch_name}-test" if is_test else release_branch_name + upcoming_version_branch = f"master-update-version-{uuid.uuid4()}" + g = Github(github_token) repository = g.get_repo(f"{REPO_OWNER}/{project_name}") newly_created_sql_file = "" - upcoming_version_branch = f"master-update-version-{uuid.uuid4()}" + # if major release if is_major_release(project_version): print(f"### {project_version} is a major release. Executing Major release flow### ") # create release-X-Y branch print(f"### Preparing {release_branch_name}...### ") # checkout master - print(f"### Checking out {main_branch}...### ") - run(f"git checkout {main_branch}") - run(f"git pull") - print(f"### OK {main_branch} checked out and pulled### ") + checkout_branch(main_branch, is_test) # create release branch in release-X.Y format - print(f"### Creating release branch with name {release_branch_name}...### ") - run(f'git checkout -b {release_branch_name}') - print(f"### OK {release_branch_name} created### ") + create_branch(release_branch_name) resource_status = ResourceStatus.RELEASE_BRANCH_LOCAL # change version info in configure.in file - print(f"### Updating version on file {configure_in_path}...### ") - if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, - f"AC_INIT([Citus], [{project_version}])"): - raise ValueError(f"{configure_in_path} does not have match for version") - print(f"### OK {configure_in_path} file is updated with project version {project_version}.### ") + update_version_in_configure_in(configure_in_path, devel_version) # execute "autoconf -f" - print(f"### Executing autoconf -f command...### ") - run("autoconf -f") - print(f"### OK autoconf -f executed.### ") + execute_autoconf_f() # change version info in multi_extension.out - print(f"### Updating version on file {multi_extension_out_path}...### ") - if not replace_line_in_file(multi_extension_out_path, MULT_EXT_MAJOR_SEARCH_PATTERN, - f" {project_version}"): - raise ValueError(f"{multi_extension_out_path} does not have match for version") - print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.### ") + update_version_in_multi_extension_out(multi_extension_out_path, devel_version) # commit all changes - print(f"### Committing changes for branch {release_branch_name}...### ") - run(f' git commit -a -m "Bump {project_name} version to {project_version} "') - print(f"### OK Changes committed for {release_branch_name}### ") + commit_all_changes_on_branch(project_name, project_version, release_branch_name) # push release branch (No PR creation!!!) if not is_test: - print(f"### Pushing changes for {release_branch_name} into remote origin...### ") - run(f"git push --set-upstream origin {release_branch_name}") + push_branch(release_branch_name) resource_status = ResourceStatus.RELEASE_BRANCH_REMOTE - print(f"### OK Changes pushed for {release_branch_name}### ") print(f"### OK {release_branch_name} prepared.### ") - # Increase version number and change upcoming version print(f"### Preparing upcoming version branch for the new master with name {upcoming_version_branch}...### ") # checkout master - print(f"### Checking out {main_branch}...### ") - run(f"git checkout {main_branch}") - print(f"### OK {main_branch} checked out.### ") + checkout_branch(main_branch, is_test) # create master-update-version-$curtime branch - print(f"### Checking out {upcoming_version_branch}...### ") - run(f"git checkout -b {upcoming_version_branch}") - print(f"### {upcoming_version_branch} checked out.### ") + create_branch(upcoming_version_branch) resource_status = ResourceStatus.UPCOMING_VERSION_LOCAL # update version info with upcoming version on configure.in - print(f"### Updating {configure_in_path} file with the upcoming version {devel_version}...### ") - if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, - f"AC_INIT([Citus], [{devel_version}])"): - raise ValueError(f"{configure_in_path} does not have match for version") - print(f"### {configure_in_path} file updated with the upcoming version {devel_version}...### ") + update_version_in_configure_in(configure_in_path, devel_version) # update version info with upcoming version on config.py - print(f"### Updating {config_py_path} file with the upcoming version {upcoming_patch_version}...### ") - if not replace_line_in_file(config_py_path, "^MASTER_VERSION =*", - f"MASTER_VERSION = '{upcoming_patch_version}'"): - raise ValueError(f"{config_py_path} does not have match for version") - print(f"### {config_py_path} file updated with the upcoming version {upcoming_patch_version}...### ") + update_version_with_upcoming_version_inconfig_py(config_py_path, upcoming_minor_version) # execute autoconf -f - print(f"### Executing autoconf -f command...### ") - run("autoconf -f") - print(f"### OK autoconf -f executed.### ") + execute_autoconf_f() # update version info with upcoming version on multiextension.out # TODO May add a special version descriptor to address lines directly to be replaced - print(f"### Updating {multi_extension_out_path} file with the upcoming version {devel_version}...### ") - if not replace_line_in_file(multi_extension_out_path, MULT_EXT_MAJOR_SEARCH_PATTERN, - f" {devel_version}"): - raise ValueError(f"{multi_extension_out_path} does not have match for version") - print(f"### OK {multi_extension_out_path} file updated with the upcoming version {devel_version}...### ") + update_version_in_multi_extension_out(multi_extension_out_path, devel_version) # get current schema version from citus.control - print(f"### Reading current schema version from {citus_control_file_path}...### ") - current_schema_version = get_current_schema_version(citus_control_file_path) - if len(current_schema_version) == 0: - raise ValueError("Version info could not be found in citus.control file") - - print(f"### OK Current schema version is {current_schema_version}### ") + current_schema_version = get_current_schema_from_citus_control(citus_control_file_path) # find current schema version info and update it with upcoming version in multi_extension.sql file - print( - f"### Updating schema version {current_schema_version} on {multi_extension_sql_path} " - f"file with the upcoming version {upcoming_version}...### ") - # TODO Append instead of replace may require - if not replace_line_in_file(multi_extension_sql_path, - f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", - f"ALTER EXTENSION citus UPDATE TO '{upcoming_patch_version}-1';"): - raise ValueError(f"{multi_extension_sql_path} does not have match for version") - print(f"### OK Current schema version updated on {multi_extension_sql_path} to {upcoming_patch_version}### ") + update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, + multi_extension_sql_path, + upcoming_minor_version, upcoming_version) # find current schema version info and update it with upcoming version in multi_extension.out file - print( - f"### Updating schema version {current_schema_version} on {multi_extension_out_path} " - f"file with the upcoming version {upcoming_version}...### ") - if not replace_line_in_file(multi_extension_out_path, - f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", - f"ALTER EXTENSION citus UPDATE TO '{upcoming_patch_version}-1';"): - raise ValueError(f"{multi_extension_out_path} does not have match for version") - print(f"### OK Current schema version updated on {multi_extension_out_path} to {upcoming_patch_version}### ") + update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, + multi_extension_out_path, + upcoming_minor_version, upcoming_version) # create new sql file with the name ">./src/backend/distributed/citus--$current_schema_version-- # $upcoming_minor_version-1.sql" - newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_patch_version}-1.sql" - print(f"### Creating file {newly_created_sql_file}...### ") - with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: - content = f"/* citus--{current_schema_version}--{upcoming_patch_version}-1 */" - content = content + "\n\n" - content = content + f"-- bump version to {upcoming_patch_version}-1" + "\n\n" - f_writer.write(content) - run(f"git add {distributed_dir_path}/{newly_created_sql_file}") - print(f"### OK {newly_created_sql_file} created.") + newly_created_sql_file = create_new_upcoming_version_sql_file(current_schema_version, distributed_dir_path, + upcoming_minor_version) # change version in citus.control file - print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_patch_version}...### ") - if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, - f"default_version = '{upcoming_patch_version}-1'"): - raise ValueError(f"{citus_control_file_path} does not have match for version") - print(f"### OK{citus_control_file_path} file updated with the upcoming version {upcoming_patch_version}...### ") + update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version) # commit and push changes on master-update-version-$curtime branch - print(f"### Committing changes for branch {upcoming_version_branch}...### ") - run(f'git commit -a -m "Bump {project_name} version to {project_version}"') - print(f"### OK Changes committed for {upcoming_version_branch}") + commit_all_changes_on_branch(project_name, project_version, upcoming_version_branch) if not is_test: - print(f"Pushing changes for {upcoming_version_branch} into remote origin...### ") - run(f"git push --set-upstream origin {upcoming_version_branch}") + push_branch(upcoming_version_branch) resource_status = ResourceStatus.UPCOMING_VERSION_REMOTE - print(f"### OK Changes pushed for {upcoming_version_branch}### ") # create pull request if not is_test: - print(f"### Creating pull request for {upcoming_version_branch}### ") - pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, - head=upcoming_version_branch, body="") - print(f"### OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}### ") + create_pull_request_for_upcoming_version_branch(main_branch, repository, upcoming_version, + upcoming_version_branch) resource_status = ResourceStatus.PULL_REQUEST_CREATED else: print(f"### {project_version} is a patch release. Executing Patch release flow ### ") # checkout release branch in release-X.Y format - print(f"### Checking out {release_branch_name}...### ") - run(f'git checkout {release_branch_name}') - print(f"### OK {release_branch_name} checked out### ") + checkout_branch(release_branch_name, is_test) # change version info in configure.in file - print(f"### Updating {configure_in_path} file with the project version {project_version}...### ") - if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, - f"AC_INIT([Citus], [{project_version}])"): - raise ValueError(f"{configure_in_path} does not have match for version") - print(f"### OK {configure_in_path} file is updated with project version {project_version}.### ") + update_version_in_configure_in(configure_in_path, project_version) # execute "auto-conf " (Not with f flag!!!) - print(f"### Executing autoconf command...### ") - run("autoconf -f") - print(f"### OK autoconf executed.### ") + execute_autoconf_f() # change version info in multi_extension.out - print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...### ") - if not replace_line_in_file(multi_extension_out_path, MULT_EXT_PATCH_SEARCH_PATTERN, - f" {project_version}"): - raise ValueError(f"{multi_extension_out_path} does not have match for version") - print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.### ") + update_version_in_multi_extension_out(multi_extension_out_path, project_version) if cherry_pick_enabled: # list all pr's with backport labels - print( - f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}### ") - all_related_prs = get_prs(repository, earliest_pr_date, main_branch) - # get commits for selected prs with backport label - prs_with_backport = get_pr_issues_by_label(all_related_prs, "backport") - print(f"### OK {len(prs_with_backport)} PRs with backport label found. PR list is as below### ") - for pr in prs_with_backport: - print(f"No:{pr.number} Title:{pr.title}\n") - # cherrypick all commits with backport label - - print(f"Cherry-picking PRs on {release_branch_name}...") - cherry_pick_prs(prs_with_backport) - print(f"OK Cherry pick completed for all PRs on branch {release_branch_name}") - # commit all-changes - - print(f"### Committing changes for branch {release_branch_name}...### ") - run(f'git commit -a -m "{project_name} version to {project_version}"') - print(f"### OK Changes committed for {release_branch_name}### ") + list_and_cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository) + + # commit all-changes + commit_all_changes_on_branch(project_name, project_version, release_branch_name) # create and push release-$minor_version-push-$curTime branch + release_pr_branch = f"{release_branch_name}_{uuid.uuid4()}" + create_branch(release_pr_branch) if not is_test: - print(f"### Pushing changes for {release_branch_name} into remote origin...### ") - run(f"git push --set-upstream origin {release_branch_name}") - resource_status = ResourceStatus.UPCOMING_VERSION_REMOTE - print(f"### OK Changes pushed for {release_branch_name}### ") + push_branch(release_pr_branch) return release_branch_name, upcoming_version_branch, f"{DISTRIBUTED_DIR_PATH}/{newly_created_sql_file}", \ resource_status +def list_and_cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository): + print( + f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}### ") + all_related_prs = get_prs(repository, earliest_pr_date, main_branch) + # get commits for selected prs with backport label + prs_with_backport = get_prs_by_label(all_related_prs, "backport") + print(f"### OK {len(prs_with_backport)} PRs with backport label found. PR list is as below### ") + for pr in prs_with_backport: + print(f"No:{pr.number} Title:{pr.title}\n") + # cherrypick all commits with backport label + print(f"Cherry-picking PRs on {release_branch_name}...") + cherry_pick_prs(prs_with_backport) + print(f"OK Cherry pick completed for all PRs on branch {release_branch_name}") + + +def create_pull_request_for_upcoming_version_branch(main_branch, repository, upcoming_version, + upcoming_version_branch): + print(f"### Creating pull request for {upcoming_version_branch}### ") + pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, + head=upcoming_version_branch, body="") + print(f"### OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}### ") + + +def push_branch(upcoming_version_branch): + print(f"Pushing changes for {upcoming_version_branch} into remote origin...### ") + run(f"git push --set-upstream origin {upcoming_version_branch}") + print(f"### OK Changes pushed for {upcoming_version_branch}### ") + + +def commit_all_changes_on_branch(project_name, project_version, release_branch): + print(f"### Committing changes for branch {release_branch}...### ") + run(f' git commit -a -m "Bump {project_name} version to {project_version} "') + print(f"### OK Changes committed for {release_branch}### ") + + +def update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version): + print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}...### ") + if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, + f"default_version = '{upcoming_minor_version}-1'"): + raise ValueError(f"{citus_control_file_path} does not have match for version") + print(f"### OK{citus_control_file_path} file updated with the upcoming version {upcoming_minor_version}...### ") + + +def update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, + multi_extension_sql_path, + upcoming_minor_version, upcoming_version): + print( + f"### Updating schema version {current_schema_version} on {multi_extension_sql_path} " + f"file with the upcoming version {upcoming_version}...### ") + # TODO Append instead of replace may require + if not replace_line_in_file(multi_extension_sql_path, + f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", + f"ALTER EXTENSION citus UPDATE TO '{upcoming_minor_version}-1';"): + raise ValueError(f"{multi_extension_sql_path} does not have match for version") + print(f"### OK Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}### ") + + +def get_current_schema_from_citus_control(citus_control_file_path): + print(f"### Reading current schema version from {citus_control_file_path}...### ") + current_schema_version = get_current_schema_version(citus_control_file_path) + if len(current_schema_version) == 0: + raise ValueError("Version info could not be found in citus.control file") + print(f"### OK Current schema version is {current_schema_version}### ") + return current_schema_version + + +def update_version_with_upcoming_version_inconfig_py(config_py_path, upcoming_minor_version): + print(f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}...### ") + if not replace_line_in_file(config_py_path, "^MASTER_VERSION =*", + f"MASTER_VERSION = '{upcoming_minor_version}'"): + raise ValueError(f"{config_py_path} does not have match for version") + print(f"### {config_py_path} file updated with the upcoming version {upcoming_minor_version}...### ") + + +def update_version_in_multi_extension_out(multi_extension_out_path, project_version): + print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...### ") + if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_MAJOR_SEARCH_PATTERN, + f" {project_version}"): + raise ValueError(f"{multi_extension_out_path} does not have match for version") + print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.### ") + + +def execute_autoconf_f(): + print(f"### Executing autoconf -f command...### ") + run("autoconf -f") + print(f"### OK autoconf -f executed.### ") + + +def update_version_in_configure_in(configure_in_path, project_version): + print(f"### Updating version on file {configure_in_path}...### ") + if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, + f"AC_INIT([Citus], [{project_version}])"): + raise ValueError(f"{configure_in_path} does not have match for version") + print(f"### OK {configure_in_path} file is updated with project version {project_version}.### ") + + +def create_branch(release_branch_name): + print(f"### Creating release branch with name {release_branch_name}...### ") + run(f'git checkout -b {release_branch_name}') + print(f"### OK {release_branch_name} created### ") + + +def checkout_branch(branch_name, is_test): + print(f"### Checking out {branch_name}...### ") + run(f"git checkout {branch_name}") + if not is_test: + run(f"git pull") + print(f"### OK {branch_name} checked out and pulled### ") + + +def create_new_upcoming_version_sql_file(current_schema_version, distributed_dir_path, + upcoming_minor_version): + newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" + print(f"### Creating file {newly_created_sql_file}...### ") + with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: + content = f"/* citus--{current_schema_version}--{upcoming_minor_version}-1 */" + content = content + "\n\n" + content = content + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" + f_writer.write(content) + run(f"git add {distributed_dir_path}/{newly_created_sql_file}") + print(f"### OK {newly_created_sql_file} created.") + return newly_created_sql_file + + def get_current_schema_version(citus_control_file_path): current_schema_version = "" with open(citus_control_file_path, "r") as cc_reader: diff --git a/python/tests/test_common_tool_methods.py b/python/tests/test_common_tool_methods.py index aacd2081..eedb97cd 100644 --- a/python/tests/test_common_tool_methods.py +++ b/python/tests/test_common_tool_methods.py @@ -72,7 +72,7 @@ def test_getprs_with_backlog_label(self): repository = g.get_repo(f"citusdata/citus") prs = get_prs(repository, datetime.strptime('2021.02.20', '%Y.%m.%d'), "master", datetime.strptime('2021.02.27', '%Y.%m.%d')) - prs_backlog = get_pr_issues_by_label(prs, "backport") + prs_backlog = get_prs_by_label(prs, "backport") self.assertEqual(1, len(prs_backlog)) self.assertEqual(4746, prs_backlog[0].number) diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index f0cc61f2..73d18bef 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -19,52 +19,53 @@ def initialize_env(): def test_major_release(): initialize_env() os.chdir("citus") - - release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( - github_token=github_token, - project_name="citus", - project_version="10.2.0", - main_branch=MAIN_BRANCH, - earliest_pr_date=datetime.strptime( - '2021.03.25 00:00', - '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, - is_test=True) - - run(f"git checkout {release_branch}") - - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.0") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.0])") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.0'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2.0 to adapt to many kinds of systems.") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.0:";;') - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") - - run(f"git checkout {upcoming_version_branch}") - - assert has_file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") - assert line_count_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 - assert has_file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert os.path.exists(f"{TEST_BASE_PATH}/{newly_created_sql_file}") - # - run(f"git checkout {MAIN_BRANCH}") - clear_branches(release_branch, resource_status, upcoming_version_branch) - clear_env() + try: + release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( + github_token=github_token, + project_name="citus", + project_version="10.2.0", + main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime( + '2021.03.25 00:00', + '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, + is_test=True) + + run(f"git checkout {release_branch}") + + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + + run(f"git checkout {upcoming_version_branch}") + + assert has_file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert line_count_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 + assert has_file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert os.path.exists(f"{TEST_BASE_PATH}/{newly_created_sql_file}") + # + run(f"git checkout {MAIN_BRANCH}") + clear_branches(release_branch, resource_status, upcoming_version_branch) + finally: + clear_env() def clear_branches(release_branch, resource_status, upcoming_version_branch): @@ -81,34 +82,31 @@ def clear_branches(release_branch, resource_status, upcoming_version_branch): def test_patch_release(): initialize_env() os.chdir("citus") - - update_release(github_token=github_token, project_name="citus", project_version="10.2.0", - main_branch=MAIN_BRANCH, - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True) - - release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( - github_token=github_token, project_name="citus", project_version="10.2.1", - main_branch=MAIN_BRANCH, - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True) - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") - run(f"git checkout {MAIN_BRANCH}") - - clear_branches(release_branch, resource_status, upcoming_version_branch) - - clear_env() - - # def tearDown(self): - # self.clear_env() + try: + update_release(github_token=github_token, project_name="citus", project_version="10.2.0", + main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, is_test=True) + + release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( + github_token=github_token, project_name="citus", project_version="10.2.1", + main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, is_test=True) + assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') + assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + run(f"git checkout {MAIN_BRANCH}") + + clear_branches(release_branch, resource_status, upcoming_version_branch) + finally: + clear_env() def clear_env(): From 85dc2315bd88380cc4ece05bb10281718613e2d2 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 4 May 2021 16:33:12 +0300 Subject: [PATCH 060/106] Clarify method name has_file_include_line->file_include_line Fix unit test cases for pr selection --- python/common_tool_methods.py | 2 +- python/tests/test_common_tool_methods.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/common_tool_methods.py b/python/common_tool_methods.py index 16a3ada6..8bdbec6d 100644 --- a/python/common_tool_methods.py +++ b/python/common_tool_methods.py @@ -113,7 +113,7 @@ def get_prs_by_label(prs: List[PullRequest.PullRequest], label_name: str): return filtered_prs -def has_file_include_line(base_path: str, relative_file_path: str, line_content: str) -> bool: +def file_include_line(base_path: str, relative_file_path: str, line_content: str) -> bool: with open(f"{base_path}/{relative_file_path}", "r") as reader: content = reader.read() lines = content.splitlines() diff --git a/python/tests/test_common_tool_methods.py b/python/tests/test_common_tool_methods.py index eedb97cd..03a6121f 100644 --- a/python/tests/test_common_tool_methods.py +++ b/python/tests/test_common_tool_methods.py @@ -64,8 +64,8 @@ def test_getprs(self): repository = g.get_repo(f"citusdata/citus") prs = get_prs(repository, datetime.strptime('2021.02.26', '%Y.%m.%d'), "master", datetime.strptime('2021.03.02', '%Y.%m.%d')) - self.assertEqual(5, len(prs)) - self.assertEqual(4760, prs[0].number) + self.assertEqual(1, len(prs)) + self.assertEqual(4751, prs[0].number) def test_getprs_with_backlog_label(self): g = Github(GITHUB_TOKEN) From acf19f78c82469c786ae10774b4d65b996d60a6b Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 4 May 2021 17:07:24 +0300 Subject: [PATCH 061/106] Fix for unit tests --- python/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index e746e2a2..97d91668 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -6,7 +6,6 @@ blinker==1.4 certifi==2019.11.28 chardet==3.0.4 Click==7.0 -cloud-init==20.4 colorama==0.4.3 command-not-found==0.3 configobj==5.0.6 From 3aa3f987c3a8975047c603c743c97d2d23e04dc3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 4 May 2021 17:08:53 +0300 Subject: [PATCH 062/106] Fix for unit tests --- python/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index 97d91668..7aea8461 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -72,7 +72,6 @@ SecretStorage==2.3.1 service-identity==18.1.0 simplejson==3.16.0 six==1.14.0 -sos==4.0 ssh-import-id==5.10 systemd-python==234 testresources==2.0.1 From a956c2a02806f7474a14ad8721153ce50f24181b Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 4 May 2021 17:14:20 +0300 Subject: [PATCH 063/106] Fix unit test method rename problem --- python/tests/test_prepare_release.py | 62 ++++++++++++++-------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index 73d18bef..d4b4460b 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -33,33 +33,33 @@ def test_major_release(): run(f"git checkout {release_branch}") - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") run(f"git checkout {upcoming_version_branch}") - assert has_file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") + assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") + assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") assert line_count_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 - assert has_file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") + assert file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") assert os.path.exists(f"{TEST_BASE_PATH}/{newly_created_sql_file}") # run(f"git checkout {MAIN_BRANCH}") @@ -93,15 +93,15 @@ def test_patch_release(): main_branch=MAIN_BRANCH, earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), exec_path=TEST_BASE_PATH, is_test=True) - assert has_file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') - assert has_file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") + assert file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") + assert file_include_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') + assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") run(f"git checkout {MAIN_BRANCH}") clear_branches(release_branch, resource_status, upcoming_version_branch) From 1330096f80893ee28ffd53ee6851ca5d667815f4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 7 May 2021 18:09:15 +0300 Subject: [PATCH 064/106] Fix round 2 PR #91 comments --- python/common_tool_methods.py | 62 ++-- python/common_validations.py | 9 +- python/prepare_release.py | 321 ++++++++++-------- python/tests/test_common_tool_methods.py | 8 +- python/tests/test_prepare_release.py | 100 +++--- .../tests/test_update_package_properties.py | 2 +- python/update_package_properties.py | 12 +- 7 files changed, 285 insertions(+), 229 deletions(-) diff --git a/python/common_tool_methods.py b/python/common_tool_methods.py index 8bdbec6d..cb3841fa 100644 --- a/python/common_tool_methods.py +++ b/python/common_tool_methods.py @@ -8,6 +8,10 @@ from jinja2 import Environment, FileSystemLoader from .common_validations import * +from git import Repo +import pathlib2 + +BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] def get_spec_file_name(project_name: str) -> str: @@ -30,7 +34,7 @@ def get_template_environment(template_dir: str) -> Environment: return env -def find_nth_overlapping(subject_string, search_string, n) -> int: +def find_nth_occurrence_position(subject_string: str, search_string: str, n) -> int: start = subject_string.find(search_string) while start >= 0 and n > 1: @@ -39,17 +43,31 @@ def find_nth_overlapping(subject_string, search_string, n) -> int: return start -def find_nth_overlapping_line_by_regex(subject_string, regex_pattern, n) -> int: +def find_nth_matching_line_number(subject_string: str, regex_pattern: str, n: int) -> int: + """Takes a subject string, regex param and the search index as parameter and returns line number of found match. + If not found returns -1""" lines = subject_string.splitlines() counter = 0 - for i in range(len(lines)): - if re.match(regex_pattern, lines[i]): + for line_number, line in enumerate(lines): + if re.match(regex_pattern, line): counter = counter + 1 if counter == n: - return i + return line_number return -1 +def find_nth_matching_line(subject_string: str, regex_pattern: str, n: int) -> str: + """Takes a subject string, regex param and the search index as parameter and returns line content of found match. + If not found returns empty string""" + lines = subject_string.splitlines() + + line_number = find_nth_matching_line_number(subject_string, regex_pattern, n) + if line_number != -1: + return lines[line_number] + else: + return "" + + def remove_string_inside_parentheses(param: str) -> str: return re.sub(r"[(\[].*?[)\]]", "", param) @@ -64,7 +82,7 @@ def cherry_pick_prs(prs: List[PullRequest.PullRequest]): commits = pr.get_commits() for single_commit in commits: cp_result = run(f"git cherry-pick {single_commit.commit.sha}") - print(f"Cherry pick result for PR No {pr.number} and commit sha {single_commit.commit.sha}: {cp_result} ") + print(f"Cherry pick result for PR no {pr.number} and commit sha {single_commit.commit.sha}: {cp_result}") def get_version_details(version: is_version(str)) -> Dict[str, str]: @@ -72,10 +90,10 @@ def get_version_details(version: is_version(str)) -> Dict[str, str]: return {"major": version_parts[0], "minor": version_parts[1], "patch": version_parts[2]} -def get_default_upcoming_version(version: is_version(str)) -> str: +def get_upcoming_patch_version(version: is_version(str)) -> str: project_version_details = get_version_details(version) return f'{project_version_details["major"]}.{project_version_details["minor"]}.' \ - f'{str(int(project_version_details["patch"]) + 1)}' + f'{int(project_version_details["patch"]) + 1}' def get_upcoming_minor_version(version: is_version(str)) -> str: @@ -94,18 +112,13 @@ def str_array_to_str(str_array: List[str]) -> str: def get_prs(repo: Repository.Repository, earliest_date: datetime, base_branch: str, last_date: datetime = None): pulls = repo.get_pulls(state="all", base=base_branch, sort="created", direction="desc") - picked_pulls = [] - for pull in pulls: - if not pull.is_merged(): - continue - if pull.created_at < earliest_date: - break - if pull.merged_at is not None and earliest_date < pull.merged_at < last_date: - picked_pulls.append(pull) - return picked_pulls - - -def get_prs_by_label(prs: List[PullRequest.PullRequest], label_name: str): + filtered_pulls = [p for p in pulls if + p.is_merged() and p.created_at > earliest_date and p.merged_at is not None and + earliest_date < p.merged_at < last_date] + return filtered_pulls + + +def filter_prs_by_label(prs: List[PullRequest.PullRequest], label_name: str): filtered_prs = [] for pr in prs: if any(label.name == label_name for label in pr.labels): @@ -113,7 +126,7 @@ def get_prs_by_label(prs: List[PullRequest.PullRequest], label_name: str): return filtered_prs -def file_include_line(base_path: str, relative_file_path: str, line_content: str) -> bool: +def file_includes_line(base_path: str, relative_file_path: str, line_content: str) -> bool: with open(f"{base_path}/{relative_file_path}", "r") as reader: content = reader.read() lines = content.splitlines() @@ -123,7 +136,7 @@ def file_include_line(base_path: str, relative_file_path: str, line_content: str return False -def line_count_in_file(base_path: str, relative_file_path: str, search_line: str) -> int: +def count_line_in_file(base_path: str, relative_file_path: str, search_line: str) -> int: with open(f"{base_path}/{relative_file_path}", "r") as reader: content = reader.read() lines = content.splitlines() @@ -144,3 +157,8 @@ def replace_line_in_file(file: str, match_regex: str, replace_str: str) -> bool: writer.write(edited_content) return has_match + + +def get_current_branch() -> str: + repo = Repo(BASE_GIT_PATH) + return repo.active_branch diff --git a/python/common_validations.py b/python/common_validations.py index 9a3f181d..08432bf7 100644 --- a/python/common_validations.py +++ b/python/common_validations.py @@ -2,14 +2,15 @@ from parameters_validation import parameter_validation import re -CITUS_VERSION_PATTERN = r"\d{1,2}\.\d{1,2}\.\d{1,2}$" +CITUS_MINOR_VERSION_PATTERN = r"\d{1,2}\.\d{1,2}" +CITUS_PATCH_VERSION_PATTERN = CITUS_MINOR_VERSION_PATTERN + r".\d{1,2}" @parameter_validation def is_version(version: str): if not version: raise ValueError("version should be non-empty and should not be None") - if not re.match(CITUS_VERSION_PATTERN, version): + if not re.match(CITUS_PATCH_VERSION_PATTERN, version): raise ValueError( "version should include three level of digits separated by dots, e.g: 10.0.1") @@ -17,8 +18,8 @@ def is_version(version: str): @parameter_validation def is_tag(tag: str): if not tag: - raise ValueError("tag should be non-empty and should not be None") - if not re.match(f"v{CITUS_VERSION_PATTERN}", tag): + raise ValueError("tag should be non-empty and should not be None") + if not re.match(f"v{CITUS_PATCH_VERSION_PATTERN}", tag): raise ValueError( "tag should start with 'v' and should include three level of digits separated by dots, e.g: v10.0.1") diff --git a/python/prepare_release.py b/python/prepare_release.py index 76617bae..ad04d08b 100644 --- a/python/prepare_release.py +++ b/python/prepare_release.py @@ -1,11 +1,19 @@ import uuid +import os +import re +import datetime from enum import Enum from typing import Tuple +from git import Repo +import pathlib2 +from dataclasses import dataclass from github import Github -from .common_tool_methods import * -from .common_validations import * +from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs, + filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, + find_nth_matching_line) +from .common_validations import CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" CITUS_CONTROL = "src/backend/distributed/citus.control" @@ -16,8 +24,9 @@ CONFIGURE = "configure" CITUS_CONTROL_SEARCH_PATTERN = r"^default_version*" -MULTI_EXT_MAJOR_SEARCH_PATTERN = r"^\s*\d{1,2}\.\d{1,2}devel$" -MULTI_EXT_MINOR_PATCH_SEARCH_PATTERN = r"^\s*" + CITUS_VERSION_PATTERN +MULTI_EXT_DEVEL_SEARCH_PATTERN = r"^\s*" + CITUS_MINOR_VERSION_PATTERN + "devel$" +MULTI_EXT_PATCH_SEARCH_PATTERN = r"^\s*" + CITUS_PATCH_VERSION_PATTERN + r"$" +CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN = r"^MASTER_VERSION = '\d+.\d+'" CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" REPO_OWNER = "citusdata" @@ -32,6 +41,16 @@ class ResourceStatus(Enum): PULL_REQUEST_CREATED = 6 +@dataclass() +class UpdateReleaseReturnValue: + release_branch_name: str + upcoming_version_branch: str + upgrade_path_sql_file: str + + +BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] + + def get_minor_version(version: str) -> str: project_version_details = get_version_details(version) return f'{project_version_details["major"]}.{project_version_details["minor"]}' @@ -39,7 +58,7 @@ def get_minor_version(version: str) -> str: def update_release(github_token: str, project_name: str, project_version: is_version(str), main_branch: str, earliest_pr_date: datetime, exec_path: str, is_test: bool = False, - cherry_pick_enabled: bool = False) -> Tuple[str, str, str, ResourceStatus]: + cherry_pick_enabled: bool = False) -> UpdateReleaseReturnValue: multi_extension_sql_path = f"{exec_path}/{MULTI_EXTENSION_SQL}" citus_control_file_path = f"{exec_path}/{CITUS_CONTROL}" multi_extension_out_path = f"{exec_path}/{MULTI_EXTENSION_OUT}" @@ -47,10 +66,8 @@ def update_release(github_token: str, project_name: str, project_version: is_ver config_py_path = f"{exec_path}/{CONFIG_PY}" distributed_dir_path = f"{exec_path}/{DISTRIBUTED_DIR_PATH}" - resource_status = ResourceStatus.INITIAL - project_version_details = get_version_details(project_version) - default_upcoming_version = get_default_upcoming_version(project_version) + default_upcoming_version = get_upcoming_patch_version(project_version) upcoming_version = os.getenv("UPCOMING_VERSION", default=default_upcoming_version) upcoming_minor_version = get_minor_version(upcoming_version) devel_version = f"{upcoming_minor_version}devel" @@ -66,115 +83,138 @@ def update_release(github_token: str, project_name: str, project_version: is_ver # if major release if is_major_release(project_version): print(f"### {project_version} is a major release. Executing Major release flow### ") - # create release-X-Y branch - print(f"### Preparing {release_branch_name}...### ") - # checkout master - checkout_branch(main_branch, is_test) - # create release branch in release-X.Y format - create_branch(release_branch_name) - resource_status = ResourceStatus.RELEASE_BRANCH_LOCAL - # change version info in configure.in file - update_version_in_configure_in(configure_in_path, devel_version) - - # execute "autoconf -f" - execute_autoconf_f() - # change version info in multi_extension.out - update_version_in_multi_extension_out(multi_extension_out_path, devel_version) - # commit all changes - commit_all_changes_on_branch(project_name, project_version, release_branch_name) - # push release branch (No PR creation!!!) - if not is_test: - push_branch(release_branch_name) - resource_status = ResourceStatus.RELEASE_BRANCH_REMOTE - print(f"### OK {release_branch_name} prepared.### ") - # Increase version number and change upcoming version - - print(f"### Preparing upcoming version branch for the new master with name {upcoming_version_branch}...### ") - # checkout master - checkout_branch(main_branch, is_test) - # create master-update-version-$curtime branch - create_branch(upcoming_version_branch) - resource_status = ResourceStatus.UPCOMING_VERSION_LOCAL - # update version info with upcoming version on configure.in - update_version_in_configure_in(configure_in_path, devel_version) - # update version info with upcoming version on config.py - update_version_with_upcoming_version_inconfig_py(config_py_path, upcoming_minor_version) - # execute autoconf -f - execute_autoconf_f() - # update version info with upcoming version on multiextension.out - # TODO May add a special version descriptor to address lines directly to be replaced - update_version_in_multi_extension_out(multi_extension_out_path, devel_version) - # get current schema version from citus.control - current_schema_version = get_current_schema_from_citus_control(citus_control_file_path) - - # find current schema version info and update it with upcoming version in multi_extension.sql file - update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, - multi_extension_sql_path, - upcoming_minor_version, upcoming_version) - # find current schema version info and update it with upcoming version in multi_extension.out file - update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, - multi_extension_out_path, - upcoming_minor_version, upcoming_version) - # create new sql file with the name ">./src/backend/distributed/citus--$current_schema_version-- - # $upcoming_minor_version-1.sql" - newly_created_sql_file = create_new_upcoming_version_sql_file(current_schema_version, distributed_dir_path, - upcoming_minor_version) - # change version in citus.control file - update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version) - # commit and push changes on master-update-version-$curtime branch - commit_all_changes_on_branch(project_name, project_version, upcoming_version_branch) - if not is_test: - push_branch(upcoming_version_branch) - resource_status = ResourceStatus.UPCOMING_VERSION_REMOTE - # create pull request - if not is_test: - create_pull_request_for_upcoming_version_branch(main_branch, repository, upcoming_version, - upcoming_version_branch) - resource_status = ResourceStatus.PULL_REQUEST_CREATED + prepare_release_branch_for_major_release(configure_in_path, devel_version, is_test, main_branch, + multi_extension_out_path, + project_name, project_version, release_branch_name) + + newly_created_sql_file = prepare_upcoming_version_branch(citus_control_file_path, + config_py_path, configure_in_path, + devel_version, distributed_dir_path, + is_test, main_branch, + multi_extension_out_path, + multi_extension_sql_path, + project_name, + project_version, repository, + upcoming_minor_version, + upcoming_version, + upcoming_version_branch) + print(f"OK {project_version} Major release flow executed successfully") else: - print(f"### {project_version} is a patch release. Executing Patch release flow ### ") - # checkout release branch in release-X.Y format - checkout_branch(release_branch_name, is_test) - - # change version info in configure.in file - update_version_in_configure_in(configure_in_path, project_version) - # execute "auto-conf " (Not with f flag!!!) - execute_autoconf_f() - # change version info in multi_extension.out - update_version_in_multi_extension_out(multi_extension_out_path, project_version) - - if cherry_pick_enabled: - # list all pr's with backport labels - list_and_cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository) - - # commit all-changes - commit_all_changes_on_branch(project_name, project_version, release_branch_name) - # create and push release-$minor_version-push-$curTime branch - release_pr_branch = f"{release_branch_name}_{uuid.uuid4()}" - create_branch(release_pr_branch) - if not is_test: - push_branch(release_pr_branch) - return release_branch_name, upcoming_version_branch, f"{DISTRIBUTED_DIR_PATH}/{newly_created_sql_file}", \ - resource_status - - -def list_and_cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository): + prepare_release_branch_for_patch_release(cherry_pick_enabled, configure_in_path, earliest_pr_date, is_test, + main_branch, multi_extension_out_path, project_name, project_version, + release_branch_name, repository) + return UpdateReleaseReturnValue(release_branch_name, upcoming_version_branch, + f"{DISTRIBUTED_DIR_PATH}/{newly_created_sql_file}") + + +def prepare_release_branch_for_patch_release(cherry_pick_enabled, configure_in_path, earliest_pr_date, is_test, + main_branch, multi_extension_out_path, project_name, project_version, + release_branch_name, repository): + print(f"### {project_version} is a patch release. Executing Patch release flow ### ") + # checkout release branch (release-X.Y) + checkout_branch(release_branch_name, is_test) + # change version info in configure.in file + update_version_in_configure_in(configure_in_path, project_version) + # execute "auto-conf " + execute_autoconf_f() + # change version info in multi_extension.out + update_version_in_multi_extension_out(multi_extension_out_path, project_version) + if cherry_pick_enabled: + # cherry-pick the pr's with backport labels + cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository) + # commit all changes + commit_changes_for_version_bump(project_name, project_version) + # create and push release-$minor_version-push-$curTime branch + release_pr_branch = f"{release_branch_name}_{uuid.uuid4()}" + create_and_checkout_branch(release_pr_branch) + if not is_test: + push_branch(release_pr_branch) + + +def prepare_upcoming_version_branch(citus_control_file_path, config_py_path, configure_in_path, devel_version, + distributed_dir_path, is_test, main_branch, multi_extension_out_path, + multi_extension_sql_path, project_name, project_version, + repository, upcoming_minor_version, upcoming_version, + upcoming_version_branch): + print(f"### Preparing {upcoming_version_branch} branch that bumps master version.### ") + # checkout master + checkout_branch(main_branch, is_test) + # create master-update-version-$curtime branch + create_and_checkout_branch(upcoming_version_branch) + # update version info with upcoming version on configure.in + update_version_in_configure_in(configure_in_path, devel_version) + # update version info with upcoming version on config.py + update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_minor_version) + # execute autoconf -f + execute_autoconf_f() + # update version info with upcoming version on multiextension.out + update_version_in_multi_extension_out(multi_extension_out_path, devel_version) + # get current schema version from citus.control + current_schema_version = get_current_schema_from_citus_control(citus_control_file_path) + # find current schema version info and update it with upcoming version in multi_extension.sql file + update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, + multi_extension_sql_path, + upcoming_minor_version, upcoming_version) + # find current schema version info and update it with upcoming version in multi_extension.out file + update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, + multi_extension_out_path, + upcoming_minor_version, upcoming_version) + # create a new sql file for upgrade path: + newly_created_sql_file = create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path, + upcoming_minor_version) + # change version in citus.control file + update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version) + # commit and push changes on master-update-version-$curtime branch + commit_changes_for_version_bump(project_name, project_version) + if not is_test: + push_branch(upcoming_version_branch) + + # create pull request + create_pull_request_for_upcoming_version_branch(repository, main_branch, upcoming_version_branch, + upcoming_version) + resource_status = ResourceStatus.PULL_REQUEST_CREATED + print(f"### OK {upcoming_version_branch} prepared.### ") + return newly_created_sql_file + + +def prepare_release_branch_for_major_release(configure_in_path, devel_version, is_test, main_branch, + multi_extension_out_path, + project_name, project_version, release_branch_name): + print(f"### Preparing {release_branch_name}...### ") + # checkout master + checkout_branch(main_branch, is_test) + # create release branch in release-X.Y format + create_and_checkout_branch(release_branch_name) + # change version info in configure.in file + update_version_in_configure_in(configure_in_path, devel_version) + # execute "autoconf -f" + execute_autoconf_f() + # change version info in multi_extension.out + update_version_in_multi_extension_out(multi_extension_out_path, devel_version) + # commit all changes + commit_changes_for_version_bump(project_name, project_version) + # push release branch (No PR creation!!!) + if not is_test: + push_branch(release_branch_name) + print(f"### OK {release_branch_name} prepared.### ") + + +def cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository): print( f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}### ") - all_related_prs = get_prs(repository, earliest_pr_date, main_branch) + prs_with_earliest_date = get_prs(repository, earliest_pr_date, main_branch) # get commits for selected prs with backport label - prs_with_backport = get_prs_by_label(all_related_prs, "backport") + prs_with_backport = filter_prs_by_label(prs_with_earliest_date, "backport") print(f"### OK {len(prs_with_backport)} PRs with backport label found. PR list is as below### ") for pr in prs_with_backport: - print(f"No:{pr.number} Title:{pr.title}\n") + print(f"\tNo:{pr.number} Title:{pr.title}") # cherrypick all commits with backport label - print(f"Cherry-picking PRs on {release_branch_name}...") + print(f"Cherry-picking PRs to {release_branch_name}...") cherry_pick_prs(prs_with_backport) print(f"OK Cherry pick completed for all PRs on branch {release_branch_name}") -def create_pull_request_for_upcoming_version_branch(main_branch, repository, upcoming_version, - upcoming_version_branch): +def create_pull_request_for_upcoming_version_branch(repository, main_branch, upcoming_version_branch, upcoming_version): print(f"### Creating pull request for {upcoming_version_branch}### ") pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, head=upcoming_version_branch, body="") @@ -187,10 +227,12 @@ def push_branch(upcoming_version_branch): print(f"### OK Changes pushed for {upcoming_version_branch}### ") -def commit_all_changes_on_branch(project_name, project_version, release_branch): - print(f"### Committing changes for branch {release_branch}...### ") - run(f' git commit -a -m "Bump {project_name} version to {project_version} "') - print(f"### OK Changes committed for {release_branch}### ") +def commit_changes_for_version_bump(project_name, project_version): + current_branch = get_current_branch() + print(f"### Committing changes for branch {current_branch}...### ") + + run(f' git commit -a -m "Bump {project_name} version to {project_version} "') + print(f"### OK Changes committed for {current_branch}### ") def update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version): @@ -198,7 +240,7 @@ def update_version_with_upcoming_version_in_citus_control(citus_control_file_pat if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, f"default_version = '{upcoming_minor_version}-1'"): raise ValueError(f"{citus_control_file_path} does not have match for version") - print(f"### OK{citus_control_file_path} file updated with the upcoming version {upcoming_minor_version}...### ") + print(f"### OK {citus_control_file_path} file is updated with the upcoming version {upcoming_minor_version}...### ") def update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, @@ -215,18 +257,33 @@ def update_schema_version_with_upcoming_version_in_multi_extension_file(current_ print(f"### OK Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}### ") -def get_current_schema_from_citus_control(citus_control_file_path): - print(f"### Reading current schema version from {citus_control_file_path}...### ") - current_schema_version = get_current_schema_version(citus_control_file_path) - if len(current_schema_version) == 0: +def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: + print(f"### Reading current schema version from {citus_control_file_path}...### ") + current_schema_version = "" + with open(citus_control_file_path, "r") as cc_reader: + cc_file_content = cc_reader.read() + cc_line = find_nth_matching_line(cc_file_content, CITUS_CONTROL_SEARCH_PATTERN, 1) + schema_not_found = False + if len(cc_line) > 0: + line_parts = cc_line.split("=") + if len(line_parts) > 0: + current_schema_version = line_parts[1] + else: + schema_not_found = True + else: + schema_not_found = True + + if schema_not_found: raise ValueError("Version info could not be found in citus.control file") - print(f"### OK Current schema version is {current_schema_version}### ") + + current_schema_version = current_schema_version.strip(" '") + print(f"OK Schema version is {current_schema_version}") return current_schema_version -def update_version_with_upcoming_version_inconfig_py(config_py_path, upcoming_minor_version): +def update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_minor_version): print(f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}...### ") - if not replace_line_in_file(config_py_path, "^MASTER_VERSION =*", + if not replace_line_in_file(config_py_path, CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN, f"MASTER_VERSION = '{upcoming_minor_version}'"): raise ValueError(f"{config_py_path} does not have match for version") print(f"### {config_py_path} file updated with the upcoming version {upcoming_minor_version}...### ") @@ -234,9 +291,10 @@ def update_version_with_upcoming_version_inconfig_py(config_py_path, upcoming_mi def update_version_in_multi_extension_out(multi_extension_out_path, project_version): print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...### ") - if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_MAJOR_SEARCH_PATTERN, + if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DEVEL_SEARCH_PATTERN, f" {project_version}"): - raise ValueError(f"{multi_extension_out_path} does not have match for version") + raise ValueError( + f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.### ") @@ -254,8 +312,9 @@ def update_version_in_configure_in(configure_in_path, project_version): print(f"### OK {configure_in_path} file is updated with project version {project_version}.### ") -def create_branch(release_branch_name): - print(f"### Creating release branch with name {release_branch_name}...### ") +def create_and_checkout_branch(release_branch_name): + get_current_branch() + print(f"### Creating release branch with name {release_branch_name} from {get_current_branch()}...### ") run(f'git checkout -b {release_branch_name}') print(f"### OK {release_branch_name} created### ") @@ -268,8 +327,8 @@ def checkout_branch(branch_name, is_test): print(f"### OK {branch_name} checked out and pulled### ") -def create_new_upcoming_version_sql_file(current_schema_version, distributed_dir_path, - upcoming_minor_version): +def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path, + upcoming_minor_version): newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" print(f"### Creating file {newly_created_sql_file}...### ") with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: @@ -280,17 +339,3 @@ def create_new_upcoming_version_sql_file(current_schema_version, distributed_dir run(f"git add {distributed_dir_path}/{newly_created_sql_file}") print(f"### OK {newly_created_sql_file} created.") return newly_created_sql_file - - -def get_current_schema_version(citus_control_file_path): - current_schema_version = "" - with open(citus_control_file_path, "r") as cc_reader: - cc_file_content = cc_reader.read() - cc_lines = cc_file_content.splitlines() - for cc_line in cc_lines: - if re.match(CITUS_CONTROL_SEARCH_PATTERN, cc_line): - line_parts = cc_line.split("=") - if len(line_parts) > 0: - current_schema_version = line_parts[1] - current_schema_version = current_schema_version.strip(" '") - return current_schema_version diff --git a/python/tests/test_common_tool_methods.py b/python/tests/test_common_tool_methods.py index 03a6121f..322a8676 100644 --- a/python/tests/test_common_tool_methods.py +++ b/python/tests/test_common_tool_methods.py @@ -19,13 +19,13 @@ def test_get_version_number_with_project_name(self): self.assertEqual(get_version_number_with_project_name("citus", "10.0.3", True, 1), "10.0.3.citus-1") def test_find_nth_overlapping(self): - self.assertEqual(find_nth_overlapping("foofoo foofoo", "foofoo", 2), 7) + self.assertEqual(find_nth_occurrence_position("foofoo foofoo", "foofoo", 2), 7) def test_find_nth_overlapping_line_by_regex(self): # Two match case - self.assertEqual(find_nth_overlapping_line_by_regex("citusx\n citusx\ncitusx", "^citusx$", 2), 2) + self.assertEqual(find_nth_matching_line_number("citusx\n citusx\ncitusx", "^citusx$", 2), 2) # No match case - self.assertEqual(find_nth_overlapping_line_by_regex("citusx\n citusx\ncitusx", "^citusy$", 2), -1) + self.assertEqual(find_nth_matching_line_number("citusx\n citusx\ncitusx", "^citusy$", 2), -1) def test_is_major_release(self): self.assertEqual(True, is_major_release("10.0.0")) @@ -72,7 +72,7 @@ def test_getprs_with_backlog_label(self): repository = g.get_repo(f"citusdata/citus") prs = get_prs(repository, datetime.strptime('2021.02.20', '%Y.%m.%d'), "master", datetime.strptime('2021.02.27', '%Y.%m.%d')) - prs_backlog = get_prs_by_label(prs, "backport") + prs_backlog = filter_prs_by_label(prs, "backport") self.assertEqual(1, len(prs_backlog)) self.assertEqual(4746, prs_backlog[0].number) diff --git a/python/tests/test_prepare_release.py b/python/tests/test_prepare_release.py index d4b4460b..2716719d 100644 --- a/python/tests/test_prepare_release.py +++ b/python/tests/test_prepare_release.py @@ -1,6 +1,10 @@ import pathlib2 +import os +from datetime import datetime -from ..prepare_release import * +from ..prepare_release import (update_release, ResourceStatus, MULTI_EXTENSION_OUT, MULTI_EXTENSION_SQL, CONFIGURE, + CONFIGURE_IN, CITUS_CONTROL, CONFIG_PY) +from ..common_tool_methods import file_includes_line, count_line_in_file, run github_token = os.getenv("GH_TOKEN") @@ -20,7 +24,7 @@ def test_major_release(): initialize_env() os.chdir("citus") try: - release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( + update_release_return_value = update_release( github_token=github_token, project_name="citus", project_version="10.2.0", @@ -31,54 +35,41 @@ def test_major_release(): exec_path=TEST_BASE_PATH, is_test=True) - run(f"git checkout {release_branch}") - - assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") - assert file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - - run(f"git checkout {upcoming_version_branch}") - - assert file_include_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") - assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") - assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") - assert line_count_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, + run(f"git checkout {update_release_return_value.release_branch_name}") + + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + + run(f"git checkout {update_release_return_value.upcoming_version_branch}") + + assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 - assert file_include_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") - assert file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert os.path.exists(f"{TEST_BASE_PATH}/{newly_created_sql_file}") - # + assert file_includes_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert os.path.exists(f"{TEST_BASE_PATH}/{update_release_return_value.upgrade_path_sql_file}") run(f"git checkout {MAIN_BRANCH}") - clear_branches(release_branch, resource_status, upcoming_version_branch) finally: clear_env() -def clear_branches(release_branch, resource_status, upcoming_version_branch): - if resource_status in (ResourceStatus.RELEASE_BRANCH_LOCAL, ResourceStatus.RELEASE_BRANCH_REMOTE, - ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, - ResourceStatus.PULL_REQUEST_CREATED): - run(f"git branch -D {release_branch} ") - - if resource_status in (ResourceStatus.UPCOMING_VERSION_LOCAL, ResourceStatus.UPCOMING_VERSION_REMOTE, - ResourceStatus.PULL_REQUEST_CREATED): - run(f"git branch -D {upcoming_version_branch} ") - - def test_patch_release(): initialize_env() os.chdir("citus") @@ -88,23 +79,22 @@ def test_patch_release(): earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), exec_path=TEST_BASE_PATH, is_test=True) - release_branch, upcoming_version_branch, newly_created_sql_file, resource_status = update_release( + update_release_return_value = update_release( github_token=github_token, project_name="citus", project_version="10.2.1", main_branch=MAIN_BRANCH, earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), exec_path=TEST_BASE_PATH, is_test=True) - assert file_include_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") - assert file_include_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") - assert file_include_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') - assert file_include_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, + r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") run(f"git checkout {MAIN_BRANCH}") - clear_branches(release_branch, resource_status, upcoming_version_branch) finally: clear_env() diff --git a/python/tests/test_update_package_properties.py b/python/tests/test_update_package_properties.py index 240ac95c..208dd8ee 100644 --- a/python/tests/test_update_package_properties.py +++ b/python/tests/test_update_package_properties.py @@ -26,7 +26,7 @@ def test_get_version_number_with_project_name(): def test_find_nth_overlapping(): - assert find_nth_overlapping("foofoo foofoo", "foofoo", 2) == 7 + assert find_nth_occurrence_position("foofoo foofoo", "foofoo", 2) == 7 def test_get_changelog_for_tag(): diff --git a/python/update_package_properties.py b/python/update_package_properties.py index 544bcc60..bfdaa66c 100644 --- a/python/update_package_properties.py +++ b/python/update_package_properties.py @@ -76,11 +76,13 @@ def set_changelog_date(self, param: datetime): self.__changelog_date = param return self + def get_rpm_version(project_name: str, version: str) -> str: return f"{version}.{project_name}" + def get_last_changelog_content(all_changelog_content: str) -> str: - second_changelog_index = find_nth_overlapping(all_changelog_content, "###", 3) + second_changelog_index = find_nth_occurrence_position(all_changelog_content, "###", 3) changelogs = all_changelog_content[:second_changelog_index] lines = changelogs.splitlines() if len(lines) < 1: @@ -92,7 +94,7 @@ def get_last_changelog_content(all_changelog_content: str) -> str: def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: - second_changelog_index = find_nth_overlapping_line_by_regex(all_changelog_content, "^[a-zA-Z]", 2) + second_changelog_index = find_nth_matching_line_number(all_changelog_content, "^[a-zA-Z]", 2) lines = all_changelog_content.splitlines() changelogs = "\n".join(lines[:second_changelog_index - 1]) + "\n" if len(lines) < 1: @@ -102,7 +104,7 @@ def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: @parameter_validation def is_project_changelog_header(header: str): - if header is None or not header: + if not header: raise ValueError("header should be non-empty and should not be None") if not re.match(r"^### \w+\sv\d+\.\d+\.\d+\s\(\w+\s\d+,\s\d+\)\s###$", header): raise ValueError( @@ -117,7 +119,7 @@ def get_changelog_for_tag(github_token: str, project_name: str, tag_name: str) - return last_changelog_content -# truncates # chars , get the version an put parentheses around version number adds 'stable; urgency=low' at the end +# truncates # chars, get the version an put parentheses around version number adds 'stable; urgency=low' at the end # changelog_header=> ### citus v8.3.3 (March 23, 2021) ### # debian header => citus (10.0.3.citus-1) stable; urgency=low @validate_parameters @@ -167,7 +169,7 @@ def update_pkgvars(project_name: str, version: is_version(non_empty(no_whitespac fancy_release_count: non_negative(int), templates_path: str, pkgvars_path: str) -> None: env = get_template_environment(templates_path) - version_str = get_version_number_with_project_name(project_name,version, fancy, fancy_release_count) + version_str = get_version_number_with_project_name(project_name, version, fancy, fancy_release_count) template = env.get_template('pkgvars.tmpl') From ffb11428cce1e685ed7e906a8fd061d9379718fd Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 20 May 2021 13:54:51 +0300 Subject: [PATCH 065/106] Fix python build problem related to ubuntu advantage package --- packaging_automation/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packaging_automation/requirements.txt b/packaging_automation/requirements.txt index 7aea8461..dddf3629 100644 --- a/packaging_automation/requirements.txt +++ b/packaging_automation/requirements.txt @@ -77,7 +77,6 @@ systemd-python==234 testresources==2.0.1 toml==0.10.2 Twisted==18.9.0 -ubuntu-advantage-tools==20.3 ufw==0.36 urllib3==1.25.8 virtualenv==20.0.17 From c40558ff34781e768cefaf23b486443ddda2c929 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 20 May 2021 15:03:25 +0300 Subject: [PATCH 066/106] Add GitPython to fix build --- packaging_automation/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging_automation/requirements.txt b/packaging_automation/requirements.txt index dddf3629..025ec940 100644 --- a/packaging_automation/requirements.txt +++ b/packaging_automation/requirements.txt @@ -18,7 +18,8 @@ distro==1.4.0 distro-info===0.23ubuntu1 entrypoints==0.3 filelock==3.0.12 -httplib2==0.14.0 +GitPython==3.1.14 +httplib2==0.19.0 hyperlink==19.0.0 idna==2.8 importlib-metadata==1.5.0 From 217ccf245854e4d11beef9ef8597347c1ebec946 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 20 May 2021 15:09:48 +0300 Subject: [PATCH 067/106] Add git credentials to checkout fix prepare release tests --- .github/workflows/tool-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index d80f2a65..298d2104 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -25,6 +25,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Define git credentails + run: git config --global user.email "gindibay@microsoft.com"&& git config --global user.name "Gurkan Indibay" - name: Install package dependencies run: sudo apt install libcurl4-openssl-dev libssl-dev - name: Install python requirements From 8461511455c04caf7ef0f52dbcd39f113cfdf43f Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 20 May 2021 15:21:41 +0300 Subject: [PATCH 068/106] Add new lines into the end of files that don't have one --- .../tests/files/verify/expected_changelog_10.0.3.txt | 1 - .../tests/files/verify/rpm_latest_changelog_reference.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt b/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt index 889d3414..99814673 100644 --- a/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt +++ b/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt @@ -30,4 +30,3 @@ an open tuple store * Relaxes the locks when adding an existing node - diff --git a/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt b/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt index 6518d086..29a2d5c4 100644 --- a/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt +++ b/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt @@ -1,2 +1,2 @@ * Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 -- Official 10.0.3 release of Citus \ No newline at end of file +- Official 10.0.3 release of Citus From 1273fe22f4f24aec4cad03a726ecc35cfcf12de4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 20 May 2021 15:26:46 +0300 Subject: [PATCH 069/106] Shorten getting date from environment to make the code cleaner --- packaging_automation/tests/test_update_package_properties.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/tests/test_update_package_properties.py b/packaging_automation/tests/test_update_package_properties.py index d3bdc4cf..15e1ea19 100644 --- a/packaging_automation/tests/test_update_package_properties.py +++ b/packaging_automation/tests/test_update_package_properties.py @@ -13,8 +13,8 @@ PROJECT_NAME = os.getenv("PROJECT_NAME", default="citus") MICROSOFT_EMAIL = os.getenv("MICROSOFT_EMAIL", default="gindibay@microsoft.com") NAME_SURNAME = os.getenv("NAME_SURNAME", default="Gurkan Indibay") -CHANGELOG_DATE = datetime.strptime('Thu, 18 Mar 2021 01:40:08 +0000', '%a, %d %b %Y %H:%M:%S %z') if os.getenv( - "CHANGELOG_DATE") is None else datetime.strptime(os.getenv("CHANGELOG_DATE"), '%a, %d %b %Y %H:%M:%S %z') +CHANGELOG_DATE_STR = os.getenv("CHANGELOG_DATE", 'Thu, 18 Mar 2021 01:40:08 +0000') +CHANGELOG_DATE = datetime.strptime(CHANGELOG_DATE_STR, '%a, %d %b %Y %H:%M:%S %z') def default_changelog_param_for_test(latest_changelog, changelog_date): From 89c0e63b047b1eafcedea5a79499025fd87a8187 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 20 May 2021 15:58:49 +0300 Subject: [PATCH 070/106] Shorten getting date from environment to make the code cleaner Recover some code blocks missing while merging --- .../verify/expected_changelog_10.0.3.txt | 1 + .../verify/rpm_latest_changelog_reference.txt | 2 +- .../tests/test_common_tool_methods.py | 4 +-- .../tests/test_update_package_properties.py | 2 -- .../update_package_properties.py | 28 +++---------------- 5 files changed, 8 insertions(+), 29 deletions(-) diff --git a/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt b/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt index 99814673..889d3414 100644 --- a/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt +++ b/packaging_automation/tests/files/verify/expected_changelog_10.0.3.txt @@ -30,3 +30,4 @@ an open tuple store * Relaxes the locks when adding an existing node + diff --git a/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt b/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt index 29a2d5c4..6518d086 100644 --- a/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt +++ b/packaging_automation/tests/files/verify/rpm_latest_changelog_reference.txt @@ -1,2 +1,2 @@ * Thu Mar 18 2021 - Gurkan Indibay 10.0.3.citus-1 -- Official 10.0.3 release of Citus +- Official 10.0.3 release of Citus \ No newline at end of file diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 322a8676..cf9f1027 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -18,10 +18,10 @@ def test_get_version_number(self): def test_get_version_number_with_project_name(self): self.assertEqual(get_version_number_with_project_name("citus", "10.0.3", True, 1), "10.0.3.citus-1") - def test_find_nth_overlapping(self): + def test_find_nth_occurrence_position(self): self.assertEqual(find_nth_occurrence_position("foofoo foofoo", "foofoo", 2), 7) - def test_find_nth_overlapping_line_by_regex(self): + def test_find_nth_matching_line_number_by_regex(self): # Two match case self.assertEqual(find_nth_matching_line_number("citusx\n citusx\ncitusx", "^citusx$", 2), 2) # No match case diff --git a/packaging_automation/tests/test_update_package_properties.py b/packaging_automation/tests/test_update_package_properties.py index 15e1ea19..78dad632 100644 --- a/packaging_automation/tests/test_update_package_properties.py +++ b/packaging_automation/tests/test_update_package_properties.py @@ -36,8 +36,6 @@ def test_get_version_number_with_project_name(): assert DEFAULT_CHANGELOG_PARAM_FOR_TEST.version_number_with_project_name() == "10.0.3.citus-1" -def test_find_nth_overlapping(): - assert find_nth_overlapping("foofoo foofoo", "foofoo", 2) == 7 def test_get_changelog_for_tag(): diff --git a/packaging_automation/update_package_properties.py b/packaging_automation/update_package_properties.py index 15483707..80c11c86 100644 --- a/packaging_automation/update_package_properties.py +++ b/packaging_automation/update_package_properties.py @@ -10,6 +10,8 @@ parameter_validation) from dataclasses import dataclass +from .common_tool_methods import (find_nth_matching_line, find_nth_matching_line_number, find_nth_occurrence_position) + BASE_PATH = pathlib2.Path(__file__).parent.absolute() project_name_suffix_dict = {"citus": "citus", "citus-enterprise": "citus", @@ -98,30 +100,8 @@ def get_template_environment(template_dir: str) -> Environment: return env -def find_nth_overlapping(subject_string, search_string, n) -> int: - start = subject_string.find(search_string) - - while start >= 0 and n > 1: - start = subject_string.find(search_string, start + 1) - n -= 1 - return start - - -def find_nth_overlapping_line_by_regex(subject_string, regex_pattern, n) -> int: - lines = subject_string.splitlines() - counter = 0 - index = -1 - for i in range(len(lines)): - if re.match(regex_pattern, lines[i]): - counter = counter + 1 - if counter == n: - index = i - break - return index - - def get_last_changelog_content(all_changelog_content: str) -> str: - second_changelog_index = find_nth_overlapping(all_changelog_content, "###", 3) + second_changelog_index = find_nth_occurrence_position(all_changelog_content, "###", 3) changelogs = all_changelog_content[:second_changelog_index] lines = changelogs.splitlines() if len(lines) < 1: @@ -133,7 +113,7 @@ def get_last_changelog_content(all_changelog_content: str) -> str: def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: - second_changelog_index = find_nth_overlapping_line_by_regex(all_changelog_content, "^[a-zA-Z]", 2) + second_changelog_index = find_nth_matching_line_number(all_changelog_content, "^[a-zA-Z]", 2) lines = all_changelog_content.splitlines() changelogs = "\n".join(lines[:second_changelog_index - 1]) + "\n" if len(lines) < 1: From b89e7eba3ee234067a72268b47030b898efef20f Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 24 May 2021 14:38:59 +0300 Subject: [PATCH 071/106] Remove project_version parameter from input parameters --- bash/update_package_properties.sh | 2 +- packaging_automation/common_tool_methods.py | 4 ++++ .../tests/test_common_tool_methods.py | 11 ++++++++++- packaging_automation/update_package_properties.py | 8 +++++--- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/bash/update_package_properties.sh b/bash/update_package_properties.sh index bff8a51b..67dd354d 100644 --- a/bash/update_package_properties.sh +++ b/bash/update_package_properties.sh @@ -26,7 +26,7 @@ commit_message="Bump to ${PRJ_NAME} ${PRJ_VER}" git checkout -b "${pr_branch_name}" -python tools/python/update_package_properties.py --gh_token "${GH_TOKEN}" --prj_name "${PRJ_NAME}" --prj_ver "${PRJ_VER}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ +python tools/python/update_package_properties.py --gh_token "${GH_TOKEN}" --prj_name "${PRJ_NAME}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ --fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S %z')" --exec_path "$(pwd)" git commit -a -m "${commit_message}" diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index cb3841fa..2ed2fdd6 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -23,6 +23,10 @@ def get_version_number(version: str, fancy: bool, fancy_release_count: int) -> s return f"{version}{fancy_suffix}" +def get_project_version_from_tag_name(tag_name: is_tag(str)) -> str: + return tag_name[1:] + + def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, fancy_release_count: int) -> str: fancy_suffix = f"-{fancy_release_count}" if fancy else "" return f"{version}.{project_name}{fancy_suffix}" diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index cf9f1027..be80cd75 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -4,8 +4,13 @@ import pathlib2 from github import Github +from datetime import datetime -from ..common_tool_methods import * +from ..common_tool_methods import (get_version_number, get_version_number_with_project_name, + find_nth_occurrence_position, find_nth_matching_line_number, is_major_release, + str_array_to_str, run, remove_string_inside_parentheses, get_version_details, + replace_line_in_file, get_prs, filter_prs_by_label, + get_project_version_from_tag_name) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -31,6 +36,10 @@ def test_is_major_release(self): self.assertEqual(True, is_major_release("10.0.0")) self.assertEqual(False, is_major_release("10.0.1")) + def test_get_project_version_from_tag_name(self): + tag_name = "v10.0.3" + self.assertEqual("10.0.3", get_project_version_from_tag_name(tag_name)) + def test_str_array_to_str(self): self.assertEqual("1\n2\n3\n4\n", str_array_to_str(["1", "2", "3", "4"])) diff --git a/packaging_automation/update_package_properties.py b/packaging_automation/update_package_properties.py index 80c11c86..ed26a45f 100644 --- a/packaging_automation/update_package_properties.py +++ b/packaging_automation/update_package_properties.py @@ -10,7 +10,8 @@ parameter_validation) from dataclasses import dataclass -from .common_tool_methods import (find_nth_matching_line, find_nth_matching_line_number, find_nth_occurrence_position) +from .common_tool_methods import (find_nth_matching_line, find_nth_matching_line_number, find_nth_occurrence_position, + get_project_version_from_tag_name) BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -271,7 +272,6 @@ def update_all_changes(github_token: non_empty(non_blank(str)), package_properti parser = argparse.ArgumentParser() parser.add_argument('--gh_token') parser.add_argument('--prj_name') - parser.add_argument('--prj_ver') parser.add_argument('--tag_name') parser.add_argument('--fancy') parser.add_argument('--fancy_ver_no') @@ -285,9 +285,11 @@ def update_all_changes(github_token: non_empty(non_blank(str)), package_properti raise ValueError(f"fancy_ver_no is expected to be numeric actual value {arguments.fancy_ver_no}") exec_date = datetime.strptime(arguments.date, '%Y.%m.%d %H:%M:%S %z') + is_tag(arguments.tag_name) + prj_ver = get_project_version_from_tag_name(arguments.tag_name) package_properties = PackagePropertiesParams(project_name=arguments.prj_name, - project_version=arguments.prj_ver, fancy=arguments.fancy, + project_version=prj_ver, fancy=arguments.fancy, fancy_version_number=int(arguments.fancy_ver_no), name_surname=arguments.name, microsoft_email=arguments.email, changelog_date=exec_date) From 14159a4b81a0249935b2b2a7571680a0ed7fa683 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 25 May 2021 11:31:12 +0300 Subject: [PATCH 072/106] Fix for @hanefi's review round 1 for PR #91 --- .github/workflows/tool-tests.yml | 6 +- bash/update_package_properties.sh | 5 +- packaging_automation/prepare_release.py | 206 +++++++++++------- .../tests/test_prepare_release.py | 14 +- 4 files changed, 141 insertions(+), 90 deletions(-) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 298d2104..24816967 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -2,6 +2,8 @@ name: Tool Tests env: GH_TOKEN: ${{ secrets.GH_TOKEN }} + MICROSOFT_EMAIL: gindibay@microsoft.com + USER_NAME: Gurkan Indibay on: push: @@ -25,8 +27,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Define git credentails - run: git config --global user.email "gindibay@microsoft.com"&& git config --global user.name "Gurkan Indibay" + - name: Define git credentials + run: git config --global user.email "${MICROSOFT_EMAIL}"&& git config --global user.name "${USER_NAME}" - name: Install package dependencies run: sudo apt install libcurl4-openssl-dev libssl-dev - name: Install python requirements diff --git a/bash/update_package_properties.sh b/bash/update_package_properties.sh index 67dd354d..4998a3f6 100644 --- a/bash/update_package_properties.sh +++ b/bash/update_package_properties.sh @@ -26,7 +26,7 @@ commit_message="Bump to ${PRJ_NAME} ${PRJ_VER}" git checkout -b "${pr_branch_name}" -python tools/python/update_package_properties.py --gh_token "${GH_TOKEN}" --prj_name "${PRJ_NAME}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ +python tools/python/update_package_properties.py --gh_token "${GH_TOKEN}" --prj_name "${PRJ_NAME}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ --fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S %z')" --exec_path "$(pwd)" git commit -a -m "${commit_message}" @@ -35,4 +35,5 @@ echo "{\"title\":\"${commit_message}\", \"head\":\"${pr_branch_name}\", \"base\" git push origin "${pr_branch_name}" -curl -g -H "Accept: application/vnd.github.v3.full+json" -X POST --user "${GH_TOKEN}:x-oauth-basic" -d "{\"title\":\"${commit_message}\", \"head\":\"${pr_branch_name}\", \"base\":\"${main_branch_name}\"}" https://api.github.com/repos/citusdata/packaging/pulls +curl -g -H "Accept: application/vnd.github.v3.full+json" -X POST --user "${GH_TOKEN}:x-oauth-basic" -d \ + "{\"title\":\"${commit_message}\", \"head\":\"${pr_branch_name}\", \"base\":\"${main_branch_name}\"}" https://api.github.com/repos/citusdata/packaging/pulls diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index ad04d08b..d99fb8eb 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -1,14 +1,11 @@ -import uuid import os -import re -import datetime -from enum import Enum -from typing import Tuple -from git import Repo -import pathlib2 +import uuid from dataclasses import dataclass +from datetime import datetime +from enum import Enum -from github import Github +import pathlib2 +from github import Github, Repository from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, @@ -41,13 +38,58 @@ class ResourceStatus(Enum): PULL_REQUEST_CREATED = 6 -@dataclass() +@dataclass class UpdateReleaseReturnValue: release_branch_name: str upcoming_version_branch: str upgrade_path_sql_file: str +@dataclass +class MajorReleaseParams: + configure_in_path: str + devel_version: str + is_test: bool + main_branch: str + multi_extension_out_path: str + project_name: str + project_version: str + release_branch_name: str + + +@dataclass +class UpcomingVersionBranchParams: + citus_control_file_path: str + config_py_path: str + configure_in_path: str + devel_version: str + distributed_dir_path: str + is_test: bool + main_branch: str + multi_extension_out_path: str + multi_extension_sql_path: str + project_name: str + project_version: str + repository: Repository + upcoming_minor_version: str + upcoming_version: str + upcoming_version_branch: str + + +@dataclass +class PatchReleaseParams: + cherry_pick_enabled: bool + configure_in_path: str + earliest_pr_date: datetime + is_test: bool + main_branch: str + multi_extension_out_path: str + project_name: str + project_version: str + release_branch_name: str + repository: Repository + + BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] @@ -83,120 +125,132 @@ def update_release(github_token: str, project_name: str, project_version: is_ver # if major release if is_major_release(project_version): print(f"### {project_version} is a major release. Executing Major release flow### ") - prepare_release_branch_for_major_release(configure_in_path, devel_version, is_test, main_branch, - multi_extension_out_path, - project_name, project_version, release_branch_name) - - newly_created_sql_file = prepare_upcoming_version_branch(citus_control_file_path, - config_py_path, configure_in_path, - devel_version, distributed_dir_path, - is_test, main_branch, - multi_extension_out_path, - multi_extension_sql_path, - project_name, - project_version, repository, - upcoming_minor_version, - upcoming_version, - upcoming_version_branch) + major_release_params = MajorReleaseParams(configure_in_path=configure_in_path, devel_version=devel_version, + is_test=is_test, main_branch=main_branch, + multi_extension_out_path=multi_extension_out_path, + project_name=project_name, project_version=project_version, + release_branch_name=release_branch_name) + prepare_release_branch_for_major_release(major_release_params) + branch_params = UpcomingVersionBranchParams(project_version=project_version, + project_name=project_name, + upcoming_version=upcoming_version, + upcoming_version_branch=upcoming_version_branch, + devel_version=devel_version, is_test=is_test, + main_branch=main_branch, + citus_control_file_path=citus_control_file_path, + config_py_path=config_py_path, + configure_in_path=configure_in_path, + distributed_dir_path=distributed_dir_path, + repository=repository, + upcoming_minor_version=upcoming_minor_version, + multi_extension_out_path=multi_extension_out_path, + multi_extension_sql_path=multi_extension_sql_path) + + newly_created_sql_file = prepare_upcoming_version_branch(branch_params) print(f"OK {project_version} Major release flow executed successfully") else: - prepare_release_branch_for_patch_release(cherry_pick_enabled, configure_in_path, earliest_pr_date, is_test, - main_branch, multi_extension_out_path, project_name, project_version, - release_branch_name, repository) + patch_release_params = PatchReleaseParams(cherry_pick_enabled=cherry_pick_enabled, + configure_in_path=configure_in_path, + earliest_pr_date=earliest_pr_date, is_test=is_test, + main_branch=main_branch, + multi_extension_out_path=multi_extension_out_path, + project_name=project_name, project_version=project_version, + release_branch_name=release_branch_name, repository=repository) + prepare_release_branch_for_patch_release(patch_release_params) return UpdateReleaseReturnValue(release_branch_name, upcoming_version_branch, f"{DISTRIBUTED_DIR_PATH}/{newly_created_sql_file}") -def prepare_release_branch_for_patch_release(cherry_pick_enabled, configure_in_path, earliest_pr_date, is_test, - main_branch, multi_extension_out_path, project_name, project_version, - release_branch_name, repository): - print(f"### {project_version} is a patch release. Executing Patch release flow ### ") +def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleaseParams): + print(f"### {patchReleaseParams.project_version} is a patch release. Executing Patch release flow ### ") # checkout release branch (release-X.Y) - checkout_branch(release_branch_name, is_test) + checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) # change version info in configure.in file - update_version_in_configure_in(configure_in_path, project_version) + update_version_in_configure_in(patchReleaseParams.configure_in_path, patchReleaseParams.project_version) # execute "auto-conf " execute_autoconf_f() # change version info in multi_extension.out - update_version_in_multi_extension_out(multi_extension_out_path, project_version) - if cherry_pick_enabled: + update_version_in_multi_extension_out(patchReleaseParams.multi_extension_out_path, + patchReleaseParams.project_version) + if patchReleaseParams.cherry_pick_enabled: # cherry-pick the pr's with backport labels - cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository) + cherrypick_prs_with_backport_labels(patchReleaseParams.earliest_pr_date, patchReleaseParams.main_branch, + patchReleaseParams.release_branch_name, patchReleaseParams.repository) # commit all changes - commit_changes_for_version_bump(project_name, project_version) + commit_changes_for_version_bump(patchReleaseParams.project_name, patchReleaseParams.project_version) # create and push release-$minor_version-push-$curTime branch - release_pr_branch = f"{release_branch_name}_{uuid.uuid4()}" + release_pr_branch = f"{patchReleaseParams.release_branch_name}_{uuid.uuid4()}" create_and_checkout_branch(release_pr_branch) - if not is_test: + if not patchReleaseParams.is_test: push_branch(release_pr_branch) -def prepare_upcoming_version_branch(citus_control_file_path, config_py_path, configure_in_path, devel_version, - distributed_dir_path, is_test, main_branch, multi_extension_out_path, - multi_extension_sql_path, project_name, project_version, - repository, upcoming_minor_version, upcoming_version, - upcoming_version_branch): - print(f"### Preparing {upcoming_version_branch} branch that bumps master version.### ") +def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams): + print(f"### Preparing {upcoming_params.upcoming_version_branch} branch that bumps master version.### ") # checkout master - checkout_branch(main_branch, is_test) + checkout_branch(upcoming_params.main_branch, upcoming_params.is_test) # create master-update-version-$curtime branch - create_and_checkout_branch(upcoming_version_branch) + create_and_checkout_branch(upcoming_params.upcoming_version_branch) # update version info with upcoming version on configure.in - update_version_in_configure_in(configure_in_path, devel_version) + update_version_in_configure_in(upcoming_params.configure_in_path, upcoming_params.devel_version) # update version info with upcoming version on config.py - update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_minor_version) + update_version_with_upcoming_version_in_config_py(upcoming_params.config_py_path, + upcoming_params.upcoming_minor_version) # execute autoconf -f execute_autoconf_f() # update version info with upcoming version on multiextension.out - update_version_in_multi_extension_out(multi_extension_out_path, devel_version) + update_version_in_multi_extension_out(upcoming_params.multi_extension_out_path, upcoming_params.devel_version) # get current schema version from citus.control - current_schema_version = get_current_schema_from_citus_control(citus_control_file_path) + current_schema_version = get_current_schema_from_citus_control(upcoming_params.citus_control_file_path) # find current schema version info and update it with upcoming version in multi_extension.sql file update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, - multi_extension_sql_path, - upcoming_minor_version, upcoming_version) + upcoming_params.multi_extension_sql_path, + upcoming_params.upcoming_minor_version, + upcoming_params.upcoming_version) # find current schema version info and update it with upcoming version in multi_extension.out file update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, - multi_extension_out_path, - upcoming_minor_version, upcoming_version) + upcoming_params.multi_extension_out_path, + upcoming_params.upcoming_minor_version, + upcoming_params.upcoming_version) # create a new sql file for upgrade path: - newly_created_sql_file = create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path, - upcoming_minor_version) + newly_created_sql_file = create_new_sql_for_upgrade_path(current_schema_version, + upcoming_params.distributed_dir_path, + upcoming_params.upcoming_minor_version) # change version in citus.control file - update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version) + update_version_with_upcoming_version_in_citus_control(upcoming_params.citus_control_file_path, + upcoming_params.upcoming_minor_version) # commit and push changes on master-update-version-$curtime branch - commit_changes_for_version_bump(project_name, project_version) - if not is_test: - push_branch(upcoming_version_branch) + commit_changes_for_version_bump(upcoming_params.project_name, upcoming_params.project_version) + if not upcoming_params.is_test: + push_branch(upcoming_params.upcoming_version_branch) # create pull request - create_pull_request_for_upcoming_version_branch(repository, main_branch, upcoming_version_branch, - upcoming_version) + create_pull_request_for_upcoming_version_branch(upcoming_params.repository, upcoming_params.main_branch, + upcoming_params.upcoming_version_branch, + upcoming_params.upcoming_version) resource_status = ResourceStatus.PULL_REQUEST_CREATED - print(f"### OK {upcoming_version_branch} prepared.### ") + print(f"### OK {upcoming_params.upcoming_version_branch} prepared.### ") return newly_created_sql_file -def prepare_release_branch_for_major_release(configure_in_path, devel_version, is_test, main_branch, - multi_extension_out_path, - project_name, project_version, release_branch_name): - print(f"### Preparing {release_branch_name}...### ") +def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleaseParams): + print(f"### Preparing {majorReleaseParams.release_branch_name}...### ") # checkout master - checkout_branch(main_branch, is_test) + checkout_branch(majorReleaseParams.main_branch, majorReleaseParams.is_test) # create release branch in release-X.Y format - create_and_checkout_branch(release_branch_name) + create_and_checkout_branch(majorReleaseParams.release_branch_name) # change version info in configure.in file - update_version_in_configure_in(configure_in_path, devel_version) + update_version_in_configure_in(majorReleaseParams.configure_in_path, majorReleaseParams.devel_version) # execute "autoconf -f" execute_autoconf_f() # change version info in multi_extension.out - update_version_in_multi_extension_out(multi_extension_out_path, devel_version) + update_version_in_multi_extension_out(majorReleaseParams.multi_extension_out_path, majorReleaseParams.devel_version) # commit all changes - commit_changes_for_version_bump(project_name, project_version) + commit_changes_for_version_bump(majorReleaseParams.project_name, majorReleaseParams.project_version) # push release branch (No PR creation!!!) - if not is_test: - push_branch(release_branch_name) - print(f"### OK {release_branch_name} prepared.### ") + if not majorReleaseParams.is_test: + push_branch(majorReleaseParams.release_branch_name) + print(f"### OK {majorReleaseParams.release_branch_name} prepared.### ") def cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository): diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 2716719d..ebbd5bf7 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -25,15 +25,9 @@ def test_major_release(): os.chdir("citus") try: update_release_return_value = update_release( - github_token=github_token, - project_name="citus", - project_version="10.2.0", - main_branch=MAIN_BRANCH, - earliest_pr_date=datetime.strptime( - '2021.03.25 00:00', - '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, - is_test=True) + github_token=github_token, project_name="citus", project_version="10.2.0", main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=TEST_BASE_PATH, is_test=True) run(f"git checkout {update_release_return_value.release_branch_name}") @@ -102,5 +96,5 @@ def test_patch_release(): def clear_env(): if os.path.exists("../citus"): os.chdir("..") - run("chmod -R 777 citus") + # run("chmod -R 777 citus") run("sudo rm -rf citus") From 551fade54dbb4c333c88a8e208416db6265962f7 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 26 May 2021 18:23:58 +0300 Subject: [PATCH 073/106] Fix for first part of comments of @onur for PR #91 --- bash/prepare_release.sh | 0 packaging_automation/common_tool_methods.py | 59 +++++----- packaging_automation/prepare_release.py | 93 +++++++--------- packaging_automation/publish-packages.py | 103 ------------------ .../tests/test_common_tool_methods.py | 29 ++--- .../tests/test_prepare_release.py | 5 +- .../tests/test_update_package_properties.py | 11 +- .../update_package_properties.py | 5 +- 8 files changed, 93 insertions(+), 212 deletions(-) delete mode 100644 bash/prepare_release.sh delete mode 100644 packaging_automation/publish-packages.py diff --git a/bash/prepare_release.sh b/bash/prepare_release.sh deleted file mode 100644 index e69de29b..00000000 diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 2ed2fdd6..325008c1 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -7,9 +7,10 @@ from github import Repository, PullRequest from jinja2 import Environment, FileSystemLoader -from .common_validations import * +from .common_validations import (is_tag, is_version) from git import Repo import pathlib2 +from typing import Tuple BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] @@ -18,20 +19,10 @@ def get_spec_file_name(project_name: str) -> str: return f"{project_name}.spec" -def get_version_number(version: str, fancy: bool, fancy_release_count: int) -> str: - fancy_suffix = f"-{fancy_release_count}" if fancy else "" - return f"{version}{fancy_suffix}" - - def get_project_version_from_tag_name(tag_name: is_tag(str)) -> str: return tag_name[1:] -def get_version_number_with_project_name(project_name: str, version: str, fancy: bool, fancy_release_count: int) -> str: - fancy_suffix = f"-{fancy_release_count}" if fancy else "" - return f"{version}.{project_name}{fancy_suffix}" - - def get_template_environment(template_dir: str) -> Environment: file_loader = FileSystemLoader(template_dir) env = Environment(loader=file_loader) @@ -47,7 +38,7 @@ def find_nth_occurrence_position(subject_string: str, search_string: str, n) -> return start -def find_nth_matching_line_number(subject_string: str, regex_pattern: str, n: int) -> int: +def find_nth_matching_line_and_line_number(subject_string: str, regex_pattern: str, n: int) -> Tuple[int, str]: """Takes a subject string, regex param and the search index as parameter and returns line number of found match. If not found returns -1""" lines = subject_string.splitlines() @@ -56,23 +47,12 @@ def find_nth_matching_line_number(subject_string: str, regex_pattern: str, n: in if re.match(regex_pattern, line): counter = counter + 1 if counter == n: - return line_number - return -1 - + return line_number, lines[line_number] + return -1, "" -def find_nth_matching_line(subject_string: str, regex_pattern: str, n: int) -> str: - """Takes a subject string, regex param and the search index as parameter and returns line content of found match. - If not found returns empty string""" - lines = subject_string.splitlines() - line_number = find_nth_matching_line_number(subject_string, regex_pattern, n) - if line_number != -1: - return lines[line_number] - else: - return "" - - -def remove_string_inside_parentheses(param: str) -> str: +def remove_text_with_parenthesis(param: str) -> str: + """Removes texts within parentheses i.e. outside parenthesis(inside parenthesis)-> outside parenthesis """ return re.sub(r"[(\[].*?[)\]]", "", param) @@ -114,12 +94,25 @@ def str_array_to_str(str_array: List[str]) -> str: return f"{os.linesep.join(str_array)}{os.linesep}" -def get_prs(repo: Repository.Repository, earliest_date: datetime, base_branch: str, last_date: datetime = None): - pulls = repo.get_pulls(state="all", base=base_branch, sort="created", direction="desc") - filtered_pulls = [p for p in pulls if - p.is_merged() and p.created_at > earliest_date and p.merged_at is not None and - earliest_date < p.merged_at < last_date] - return filtered_pulls +def get_prs_for_patch_release(repo: Repository.Repository, earliest_date: datetime, base_branch: str, + last_date: datetime = None): + pull_requests = repo.get_pulls(state="closed", base=base_branch, sort="created", direction="desc") + + # filter pull requests according to given time interval + filtered_pull_requests = list() + for pull_request in pull_requests: + if not pull_request.is_merged(): + continue + if pull_request.merged_at < earliest_date: + continue + if last_date and pull_request.merged_at > last_date: + continue + + filtered_pull_requests.append(pull_request) + + # finally, sort the pr's by their merge date + sorted_pull_requests = sorted(filtered_pull_requests, key=lambda p: p.merged_at) + return sorted_pull_requests def filter_prs_by_label(prs: List[PullRequest.PullRequest], label_name: str): diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index d99fb8eb..b18f3be7 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -2,14 +2,14 @@ import uuid from dataclasses import dataclass from datetime import datetime -from enum import Enum import pathlib2 from github import Github, Repository -from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs, +from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, + get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, - find_nth_matching_line) + find_nth_matching_line_and_line_number) from .common_validations import CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -23,21 +23,12 @@ MULTI_EXT_DEVEL_SEARCH_PATTERN = r"^\s*" + CITUS_MINOR_VERSION_PATTERN + "devel$" MULTI_EXT_PATCH_SEARCH_PATTERN = r"^\s*" + CITUS_PATCH_VERSION_PATTERN + r"$" -CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN = r"^MASTER_VERSION = '\d+.\d+'" +CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN = r"^MASTER_VERSION = '\d+\.\d+'" CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" REPO_OWNER = "citusdata" -class ResourceStatus(Enum): - INITIAL = 1 - RELEASE_BRANCH_LOCAL = 2 - RELEASE_BRANCH_REMOTE = 3 - UPCOMING_VERSION_LOCAL = 4 - UPCOMING_VERSION_REMOTE = 5 - PULL_REQUEST_CREATED = 6 - - @dataclass class UpdateReleaseReturnValue: release_branch_name: str @@ -124,7 +115,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver # if major release if is_major_release(project_version): - print(f"### {project_version} is a major release. Executing Major release flow### ") + print(f"### {project_version} is a major release. Executing Major release flow###") major_release_params = MajorReleaseParams(configure_in_path=configure_in_path, devel_version=devel_version, is_test=is_test, main_branch=main_branch, multi_extension_out_path=multi_extension_out_path, @@ -162,7 +153,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleaseParams): - print(f"### {patchReleaseParams.project_version} is a patch release. Executing Patch release flow ### ") + print(f"### {patchReleaseParams.project_version} is a patch release. Executing Patch release flow ###") # checkout release branch (release-X.Y) checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) # change version info in configure.in file @@ -186,7 +177,7 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams): - print(f"### Preparing {upcoming_params.upcoming_version_branch} branch that bumps master version.### ") + print(f"### Preparing {upcoming_params.upcoming_version_branch} branch that bumps master version.###") # checkout master checkout_branch(upcoming_params.main_branch, upcoming_params.is_test) # create master-update-version-$curtime branch @@ -228,38 +219,38 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams create_pull_request_for_upcoming_version_branch(upcoming_params.repository, upcoming_params.main_branch, upcoming_params.upcoming_version_branch, upcoming_params.upcoming_version) - resource_status = ResourceStatus.PULL_REQUEST_CREATED - print(f"### OK {upcoming_params.upcoming_version_branch} prepared.### ") + print(f"### OK {upcoming_params.upcoming_version_branch} prepared.###") return newly_created_sql_file def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleaseParams): - print(f"### Preparing {majorReleaseParams.release_branch_name}...### ") + print(f"### Preparing {majorReleaseParams.release_branch_name}...###") # checkout master checkout_branch(majorReleaseParams.main_branch, majorReleaseParams.is_test) # create release branch in release-X.Y format create_and_checkout_branch(majorReleaseParams.release_branch_name) # change version info in configure.in file - update_version_in_configure_in(majorReleaseParams.configure_in_path, majorReleaseParams.devel_version) + update_version_in_configure_in(majorReleaseParams.configure_in_path, majorReleaseParams.project_version) # execute "autoconf -f" execute_autoconf_f() # change version info in multi_extension.out - update_version_in_multi_extension_out(majorReleaseParams.multi_extension_out_path, majorReleaseParams.devel_version) + update_version_in_multi_extension_out(majorReleaseParams.multi_extension_out_path, + majorReleaseParams.project_version) # commit all changes commit_changes_for_version_bump(majorReleaseParams.project_name, majorReleaseParams.project_version) # push release branch (No PR creation!!!) if not majorReleaseParams.is_test: push_branch(majorReleaseParams.release_branch_name) - print(f"### OK {majorReleaseParams.release_branch_name} prepared.### ") + print(f"### OK {majorReleaseParams.release_branch_name} prepared.###") def cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository): print( f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}### ") - prs_with_earliest_date = get_prs(repository, earliest_pr_date, main_branch) + prs_with_earliest_date = get_prs_for_patch_release(repository, earliest_pr_date, main_branch) # get commits for selected prs with backport label prs_with_backport = filter_prs_by_label(prs_with_earliest_date, "backport") - print(f"### OK {len(prs_with_backport)} PRs with backport label found. PR list is as below### ") + print(f"### OK {len(prs_with_backport)} PRs with backport label found. PR list is as below###") for pr in prs_with_backport: print(f"\tNo:{pr.number} Title:{pr.title}") # cherrypick all commits with backport label @@ -269,32 +260,32 @@ def cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_b def create_pull_request_for_upcoming_version_branch(repository, main_branch, upcoming_version_branch, upcoming_version): - print(f"### Creating pull request for {upcoming_version_branch}### ") + print(f"### Creating pull request for {upcoming_version_branch}###") pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, head=upcoming_version_branch, body="") - print(f"### OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}### ") + print(f"### OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}###") def push_branch(upcoming_version_branch): - print(f"Pushing changes for {upcoming_version_branch} into remote origin...### ") + print(f"Pushing changes for {upcoming_version_branch} into remote origin...###") run(f"git push --set-upstream origin {upcoming_version_branch}") - print(f"### OK Changes pushed for {upcoming_version_branch}### ") + print(f"### OK Changes pushed for {upcoming_version_branch}###") def commit_changes_for_version_bump(project_name, project_version): current_branch = get_current_branch() - print(f"### Committing changes for branch {current_branch}...### ") + print(f"### Committing changes for branch {current_branch}...###") run(f' git commit -a -m "Bump {project_name} version to {project_version} "') - print(f"### OK Changes committed for {current_branch}### ") + print(f"### OK Changes committed for {current_branch}###") def update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version): - print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}...### ") + print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}...###") if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, f"default_version = '{upcoming_minor_version}-1'"): raise ValueError(f"{citus_control_file_path} does not have match for version") - print(f"### OK {citus_control_file_path} file is updated with the upcoming version {upcoming_minor_version}...### ") + print(f"### OK {citus_control_file_path} file is updated with the upcoming version {upcoming_minor_version}...###") def update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, @@ -308,19 +299,20 @@ def update_schema_version_with_upcoming_version_in_multi_extension_file(current_ f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", f"ALTER EXTENSION citus UPDATE TO '{upcoming_minor_version}-1';"): raise ValueError(f"{multi_extension_sql_path} does not have match for version") - print(f"### OK Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}### ") + print(f"### OK Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}###") def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: - print(f"### Reading current schema version from {citus_control_file_path}...### ") + print(f"### Reading current schema version from {citus_control_file_path}...###") current_schema_version = "" with open(citus_control_file_path, "r") as cc_reader: cc_file_content = cc_reader.read() - cc_line = find_nth_matching_line(cc_file_content, CITUS_CONTROL_SEARCH_PATTERN, 1) + cc_line_number, cc_line = find_nth_matching_line_and_line_number(cc_file_content, CITUS_CONTROL_SEARCH_PATTERN, + 1) schema_not_found = False if len(cc_line) > 0: line_parts = cc_line.split("=") - if len(line_parts) > 0: + if len(line_parts) == 2: current_schema_version = line_parts[1] else: schema_not_found = True @@ -336,60 +328,59 @@ def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: def update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_minor_version): - print(f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}...### ") + print(f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}...###") if not replace_line_in_file(config_py_path, CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN, f"MASTER_VERSION = '{upcoming_minor_version}'"): raise ValueError(f"{config_py_path} does not have match for version") - print(f"### {config_py_path} file updated with the upcoming version {upcoming_minor_version}...### ") + print(f"### {config_py_path} file updated with the upcoming version {upcoming_minor_version}...###") def update_version_in_multi_extension_out(multi_extension_out_path, project_version): - print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...### ") + print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...###") if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DEVEL_SEARCH_PATTERN, f" {project_version}"): raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") - print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.### ") + print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.###") def execute_autoconf_f(): - print(f"### Executing autoconf -f command...### ") + print(f"### Executing autoconf -f command...###") run("autoconf -f") - print(f"### OK autoconf -f executed.### ") + print(f"### OK autoconf -f executed.###") def update_version_in_configure_in(configure_in_path, project_version): - print(f"### Updating version on file {configure_in_path}...### ") + print(f"### Updating version on file {configure_in_path}...###") if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{project_version}])"): raise ValueError(f"{configure_in_path} does not have match for version") - print(f"### OK {configure_in_path} file is updated with project version {project_version}.### ") + print(f"### OK {configure_in_path} file is updated with project version {project_version}.###") def create_and_checkout_branch(release_branch_name): - get_current_branch() - print(f"### Creating release branch with name {release_branch_name} from {get_current_branch()}...### ") + print(f"### Creating release branch with name {release_branch_name} from {get_current_branch()}...###") run(f'git checkout -b {release_branch_name}') - print(f"### OK {release_branch_name} created### ") + print(f"### OK {release_branch_name} created###") def checkout_branch(branch_name, is_test): - print(f"### Checking out {branch_name}...### ") + print(f"### Checking out {branch_name}...###") run(f"git checkout {branch_name}") if not is_test: run(f"git pull") - print(f"### OK {branch_name} checked out and pulled### ") + print(f"### OK {branch_name} checked out and pulled###") def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path, upcoming_minor_version): newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" - print(f"### Creating file {newly_created_sql_file}...### ") + print(f"### Creating file {newly_created_sql_file}...###") with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: content = f"/* citus--{current_schema_version}--{upcoming_minor_version}-1 */" content = content + "\n\n" content = content + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" f_writer.write(content) - run(f"git add {distributed_dir_path}/{newly_created_sql_file}") + # run(f"git add {distributed_dir_path}/{newly_created_sql_file}") print(f"### OK {newly_created_sql_file} created.") return newly_created_sql_file diff --git a/packaging_automation/publish-packages.py b/packaging_automation/publish-packages.py deleted file mode 100644 index e09fc9bf..00000000 --- a/packaging_automation/publish-packages.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 - -from subprocess import PIPE -import subprocess -import os -import json -import re -import time -import sys -from pprint import pprint - -ms_package_repo_map = { - "el/8": "centos-8", "el/7": "centos-7", "debian/buster": "debian-buster", "debian/jessie": "debian-jessie", - "debian/stretch": "debian-stretch", "ubuntu/bionic": "ubuntu-bionic", "ubuntu/xenial": "ubuntu-xenial", - "ubuntu/focal": "ubuntu-focal"} - - -def run(command, *args, **kwargs): - print(command) - result = subprocess.run(command, *args, check=True, shell=True, **kwargs) - print() - return result - - -result = run("repoclient repo list", stdout=PIPE) - -all_repos = json.loads(result.stdout) - -citus_repos = {} -for repo in all_repos: - if not repo["url"].startswith("citus-"): - continue - name = repo["url"][len("citus-"):] - if name in ("ubuntu", "debian"): - # Suffix distribution - name = name + "-" + repo["distribution"] - else: - # Put dash before numberrepo - name = re.sub(r"(\d+)", r"-\1", name) - citus_repos[name] = repo - -# pprint(citus_repos) -target_platform = sys.argv[1] -submission_responses = {} -print("Citus Repos") -print("Current Dir" + os.getcwd()) -pprint(citus_repos) -for package_file in os.listdir("pkgs/releases"): - - print("Target Platform is " + target_platform) - repo_platform = ms_package_repo_map[target_platform] - repo = citus_repos[repo_platform] - package_path = os.path.join("pkgs/releases", package_file) - - print("Repo Url:" + repo["url"]) - # Ensure deb packages contain the distribution, so they do not conflict - if repo["url"] in ("citus-ubuntu", "citus-debian"): - if repo["distribution"] not in package_file: - if not package_path.endswith("amd64.deb"): - raise "Package should have ended with amd64.deb: %s" % package_path - old_package_path = package_path - package_prefix = package_path[: -len("amd64.deb")] - package_path = "%s+%s_amd64.deb" % ( - package_prefix, - repo["distribution"], - ) - os.rename(old_package_path, package_path) - - # Publish packages - if os.path.isfile(package_path) and (package_file.endswith(".rpm") or package_file.endswith(".deb")): - result = run( - "repoclient package add --repoID %s %s" % (repo["id"], package_path), - stdout=PIPE, - ) - submission_responses[package_path] = json.loads(result.stdout) - print( - "Waiting for 30 seconds to avoid concurrency problems on publishing server" - ) - time.sleep(30) - -pprint(submission_responses) - -# Check if published successfully -for i in range(15): - unfinished_submissions = {} - for package_path, response in submission_responses.items(): - package_id = response["Location"].split("/")[-1] - - try: - run("repoclient package check %s" % package_id) - except Exception: - print(package_path, "was not finished publishing") - unfinished_submissions[package_path] = response - - if not unfinished_submissions: - break - submission_responses = unfinished_submissions - time.sleep(i) - -if unfinished_submissions: - print("The following packages were not published successfuly") - pprint(unfinished_submissions) - raise Exception("Some packages were not finished publishing") diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index be80cd75..099a34e5 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -6,31 +6,26 @@ from github import Github from datetime import datetime -from ..common_tool_methods import (get_version_number, get_version_number_with_project_name, - find_nth_occurrence_position, find_nth_matching_line_number, is_major_release, - str_array_to_str, run, remove_string_inside_parentheses, get_version_details, - replace_line_in_file, get_prs, filter_prs_by_label, - get_project_version_from_tag_name) +from ..common_tool_methods import ( + find_nth_occurrence_position, is_major_release, + str_array_to_str, run, remove_text_with_paranthesis, get_version_details, + replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, + get_project_version_from_tag_name, find_nth_matching_line_and_line_number) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() class CommonToolMethodsTestCases(unittest.TestCase): - def test_get_version_number(self): - self.assertEqual(get_version_number("10.0.3", True, 1), "10.0.3-1") - - def test_get_version_number_with_project_name(self): - self.assertEqual(get_version_number_with_project_name("citus", "10.0.3", True, 1), "10.0.3.citus-1") def test_find_nth_occurrence_position(self): self.assertEqual(find_nth_occurrence_position("foofoo foofoo", "foofoo", 2), 7) def test_find_nth_matching_line_number_by_regex(self): # Two match case - self.assertEqual(find_nth_matching_line_number("citusx\n citusx\ncitusx", "^citusx$", 2), 2) + self.assertEqual(find_nth_matching_line_and_line_number("citusx\n citusx\ncitusx", "^citusx$", 2)[0], 2) # No match case - self.assertEqual(find_nth_matching_line_number("citusx\n citusx\ncitusx", "^citusy$", 2), -1) + self.assertEqual(find_nth_matching_line_and_line_number("citusx\n citusx\ncitusx", "^citusy$", 2)[0], -1) def test_is_major_release(self): self.assertEqual(True, is_major_release("10.0.0")) @@ -49,7 +44,7 @@ def test_run(self): def test_remove_parentheses_from_string(self): self.assertEqual("out of parentheses ", - remove_string_inside_parentheses("out of parentheses (inside parentheses)")) + remove_text_with_paranthesis("out of parentheses (inside parentheses)")) def test_get_version_details(self): self.assertEqual({"major": "10", "minor": "0", "patch": "1"}, get_version_details("10.0.1")) @@ -71,16 +66,16 @@ def test_getprs(self): # created at is not seen on Github. Should be checked on API result g = Github(GITHUB_TOKEN) repository = g.get_repo(f"citusdata/citus") - prs = get_prs(repository, datetime.strptime('2021.02.26', '%Y.%m.%d'), "master", - datetime.strptime('2021.03.02', '%Y.%m.%d')) + prs = get_prs_for_patch_release(repository, datetime.strptime('2021.02.26', '%Y.%m.%d'), "master", + datetime.strptime('2021.03.02', '%Y.%m.%d')) self.assertEqual(1, len(prs)) self.assertEqual(4751, prs[0].number) def test_getprs_with_backlog_label(self): g = Github(GITHUB_TOKEN) repository = g.get_repo(f"citusdata/citus") - prs = get_prs(repository, datetime.strptime('2021.02.20', '%Y.%m.%d'), "master", - datetime.strptime('2021.02.27', '%Y.%m.%d')) + prs = get_prs_for_patch_release(repository, datetime.strptime('2021.02.20', '%Y.%m.%d'), "master", + datetime.strptime('2021.02.27', '%Y.%m.%d')) prs_backlog = filter_prs_by_label(prs, "backport") self.assertEqual(1, len(prs_backlog)) self.assertEqual(4746, prs_backlog[0].number) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index ebbd5bf7..ae3dac7e 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -2,7 +2,7 @@ import os from datetime import datetime -from ..prepare_release import (update_release, ResourceStatus, MULTI_EXTENSION_OUT, MULTI_EXTENSION_SQL, CONFIGURE, +from ..prepare_release import (update_release, MULTI_EXTENSION_OUT, MULTI_EXTENSION_SQL, CONFIGURE, CONFIGURE_IN, CITUS_CONTROL, CONFIG_PY) from ..common_tool_methods import file_includes_line, count_line_in_file, run @@ -73,7 +73,7 @@ def test_patch_release(): earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), exec_path=TEST_BASE_PATH, is_test=True) - update_release_return_value = update_release( + update_release( github_token=github_token, project_name="citus", project_version="10.2.1", main_branch=MAIN_BRANCH, earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), @@ -96,5 +96,4 @@ def test_patch_release(): def clear_env(): if os.path.exists("../citus"): os.chdir("..") - # run("chmod -R 777 citus") run("sudo rm -rf citus") diff --git a/packaging_automation/tests/test_update_package_properties.py b/packaging_automation/tests/test_update_package_properties.py index 78dad632..d9b5665f 100644 --- a/packaging_automation/tests/test_update_package_properties.py +++ b/packaging_automation/tests/test_update_package_properties.py @@ -1,9 +1,16 @@ import pytest import os +import re from shutil import copyfile +import pathlib2 +from datetime import datetime from .test_utils import are_strings_equal -from ..update_package_properties import * +from ..update_package_properties import (PackagePropertiesParams, changelog_for_tag, + get_last_changelog_content_from_debian, debian_changelog_header, + prepend_latest_changelog_into_debian_changelog, + convert_citus_changelog_into_rpm_changelog, spec_file_name, update_rpm_spec, + update_pkgvars, update_all_changes) TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[1]) @@ -36,8 +43,6 @@ def test_get_version_number_with_project_name(): assert DEFAULT_CHANGELOG_PARAM_FOR_TEST.version_number_with_project_name() == "10.0.3.citus-1" - - def test_get_changelog_for_tag(): changelog = changelog_for_tag(GITHUB_TOKEN, "citus", "v10.0.3") with open(f"{TEST_BASE_PATH}/files/verify/expected_changelog_10.0.3.txt", "r") as reader: diff --git a/packaging_automation/update_package_properties.py b/packaging_automation/update_package_properties.py index ed26a45f..345904b0 100644 --- a/packaging_automation/update_package_properties.py +++ b/packaging_automation/update_package_properties.py @@ -10,7 +10,7 @@ parameter_validation) from dataclasses import dataclass -from .common_tool_methods import (find_nth_matching_line, find_nth_matching_line_number, find_nth_occurrence_position, +from .common_tool_methods import (find_nth_matching_line_and_line_number, find_nth_occurrence_position, get_project_version_from_tag_name) BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -114,7 +114,8 @@ def get_last_changelog_content(all_changelog_content: str) -> str: def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: - second_changelog_index = find_nth_matching_line_number(all_changelog_content, "^[a-zA-Z]", 2) + second_changelog_index, second_changelog_line = find_nth_matching_line_and_line_number(all_changelog_content, + "^[a-zA-Z]", 2) lines = all_changelog_content.splitlines() changelogs = "\n".join(lines[:second_changelog_index - 1]) + "\n" if len(lines) < 1: From b86904163da17822da3728dc8ef6278222a6f942 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 27 May 2021 14:02:24 +0300 Subject: [PATCH 074/106] Fix for second part of comments of @onur for PR #91 --- packaging_automation/common_tool_methods.py | 25 +- packaging_automation/common_validations.py | 2 +- packaging_automation/prepare_release.py | 9 +- packaging_automation/requirements.in | 15 + packaging_automation/requirements.txt | 290 ++++++++++++------ packaging_automation/tests/test.py | 3 + .../tests/test_common_tool_methods.py | 13 +- .../tests/test_prepare_release.py | 35 ++- 8 files changed, 275 insertions(+), 117 deletions(-) create mode 100644 packaging_automation/requirements.in create mode 100644 packaging_automation/tests/test.py diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 325008c1..c9d365ce 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -4,7 +4,7 @@ from typing import Dict, List import os -from github import Repository, PullRequest +from github import Repository, PullRequest, Commit from jinja2 import Environment, FileSystemLoader from .common_validations import (is_tag, is_version) @@ -13,6 +13,7 @@ from typing import Tuple BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] +PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX = "\.\d{1,3}" def get_spec_file_name(project_name: str) -> str: @@ -65,8 +66,23 @@ def cherry_pick_prs(prs: List[PullRequest.PullRequest]): for pr in prs: commits = pr.get_commits() for single_commit in commits: - cp_result = run(f"git cherry-pick {single_commit.commit.sha}") - print(f"Cherry pick result for PR no {pr.number} and commit sha {single_commit.commit.sha}: {cp_result}") + if not is_merge_commit(single_commit): + cp_result = run(f"git cherry-pick {single_commit.commit.sha}") + print( + f"Cherry pick result for PR no {pr.number} and commit sha {single_commit.commit.sha}: {cp_result}") + + +def get_minor_version(version: str) -> str: + project_version_details = get_version_details(version) + return f'{project_version_details["major"]}.{project_version_details["minor"]}' + + +def get_patch_version_regex(version: is_version(str)): + return f"{re.escape(get_minor_version(version))}{PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX}" + + +def is_merge_commit(commit: Commit): + return len(commit.parents) <= 1 def get_version_details(version: is_version(str)) -> Dict[str, str]: @@ -76,8 +92,7 @@ def get_version_details(version: is_version(str)) -> Dict[str, str]: def get_upcoming_patch_version(version: is_version(str)) -> str: project_version_details = get_version_details(version) - return f'{project_version_details["major"]}.{project_version_details["minor"]}.' \ - f'{int(project_version_details["patch"]) + 1}' + return f'{get_upcoming_minor_version(version)}.{int(project_version_details["patch"]) + 1}' def get_upcoming_minor_version(version: is_version(str)) -> str: diff --git a/packaging_automation/common_validations.py b/packaging_automation/common_validations.py index 08432bf7..4477ed31 100644 --- a/packaging_automation/common_validations.py +++ b/packaging_automation/common_validations.py @@ -3,7 +3,7 @@ import re CITUS_MINOR_VERSION_PATTERN = r"\d{1,2}\.\d{1,2}" -CITUS_PATCH_VERSION_PATTERN = CITUS_MINOR_VERSION_PATTERN + r".\d{1,2}" +CITUS_PATCH_VERSION_PATTERN = CITUS_MINOR_VERSION_PATTERN + r"\.\d{1,2}" @parameter_validation diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index b18f3be7..b8043519 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -1,4 +1,5 @@ import os +import re import uuid from dataclasses import dataclass from datetime import datetime @@ -25,6 +26,7 @@ MULTI_EXT_PATCH_SEARCH_PATTERN = r"^\s*" + CITUS_PATCH_VERSION_PATTERN + r"$" CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN = r"^MASTER_VERSION = '\d+\.\d+'" + CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" REPO_OWNER = "citusdata" @@ -84,9 +86,7 @@ class PatchReleaseParams: BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] -def get_minor_version(version: str) -> str: - project_version_details = get_version_details(version) - return f'{project_version_details["major"]}.{project_version_details["minor"]}' + def update_release(github_token: str, project_name: str, project_version: is_version(str), main_branch: str, @@ -337,7 +337,8 @@ def update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_m def update_version_in_multi_extension_out(multi_extension_out_path, project_version): print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...###") - if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DEVEL_SEARCH_PATTERN, + if not replace_line_in_file(multi_extension_out_path, + f"{re.escape(get_minor_version(project_version))}{PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX}", f" {project_version}"): raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") diff --git a/packaging_automation/requirements.in b/packaging_automation/requirements.in new file mode 100644 index 00000000..462cd2ab --- /dev/null +++ b/packaging_automation/requirements.in @@ -0,0 +1,15 @@ +wheel +docker +GitPython +Jinja2 +pathlib2 +pycurl +PyGithub +pytest +python-string-utils +requests +urllib3 +gnupg +parameters_validation +PyYAML + diff --git a/packaging_automation/requirements.txt b/packaging_automation/requirements.txt index 025ec940..d8596c13 100644 --- a/packaging_automation/requirements.txt +++ b/packaging_automation/requirements.txt @@ -1,87 +1,203 @@ -adal==1.2.2 -appdirs==1.4.3 -attrs==19.3.0 -Automat==0.8.0 -blinker==1.4 -certifi==2019.11.28 -chardet==3.0.4 -Click==7.0 -colorama==0.4.3 -command-not-found==0.3 -configobj==5.0.6 -constantly==15.1.0 -cryptography==2.8 -dbus-python==1.2.16 -Deprecated==1.2.12 -distlib==0.3.0 -distro==1.4.0 -distro-info===0.23ubuntu1 -entrypoints==0.3 -filelock==3.0.12 -GitPython==3.1.14 -httplib2==0.19.0 -hyperlink==19.0.0 -idna==2.8 -importlib-metadata==1.5.0 -incremental==16.10.1 -iniconfig==1.1.1 -Jinja2==2.10.1 -jsonpatch==1.22 -jsonpointer==2.0 -jsonschema==3.2.0 -keyring==18.0.1 -language-selector==0.1 -launchpadlib==1.10.13 -lazr.restfulclient==0.14.2 -lazr.uri==1.0.3 -MarkupSafe==1.1.0 -more-itertools==4.2.0 -netifaces==0.10.4 -oauthlib==3.1.0 -packaging==20.9 -parameters-validation==1.2.0 -pathlib2==2.3.5 -pbr==5.5.1 -pexpect==4.6.0 -pipenv==2020.11.15 -pluggy==0.13.1 -py==1.10.0 -pyasn1==0.4.2 -pyasn1-modules==0.2.1 -pycairo==1.20.0 -pycurl==7.43.0.6 -PyGithub==1.54.1 -Pygments==2.3.1 -PyGObject==3.36.0 -PyHamcrest==1.9.0 -PyJWT==1.7.1 -pymacaroons==0.13.0 -PyNaCl==1.3.0 -pyOpenSSL==19.0.0 -pyparsing==2.4.7 -pyrsistent==0.15.5 -pyserial==3.4 -pytest==6.2.3 -python-apt==2.0.0+ubuntu0.20.4.4 -python-dateutil==2.7.3 -python-debian===0.1.36ubuntu1 -python-string-utils==1.0.0 -PyYAML==5.3.1 -requests==2.22.0 -requests-unixsocket==0.2.0 -SecretStorage==2.3.1 -service-identity==18.1.0 -simplejson==3.16.0 -six==1.14.0 -ssh-import-id==5.10 -systemd-python==234 -testresources==2.0.1 -toml==0.10.2 -Twisted==18.9.0 -ufw==0.36 -urllib3==1.25.8 -virtualenv==20.0.17 -wadllib==1.3.3 -wrapt==1.12.1 -zipp==1.0.0 -zope.interface==4.7.1 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes +# +attrs==19.3.0 \ + --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ + --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 + # via pytest +certifi==2019.11.28 \ + --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \ + --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f + # via requests +chardet==3.0.4 \ + --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 + # via requests +deprecated==1.2.12 \ + --hash=sha256:08452d69b6b5bc66e8330adde0a4f8642e969b9e1702904d137eeb29c8ffc771 \ + --hash=sha256:6d2de2de7931a968874481ef30208fd4e08da39177d61d3d4ebdf4366e7dbca1 + # via pygithub +docker==5.0.0 \ + --hash=sha256:3e8bc47534e0ca9331d72c32f2881bb13b93ded0bcdeab3c833fb7cf61c0a9a5 \ + --hash=sha256:fc961d622160e8021c10d1bcabc388c57d55fb1f917175afbe24af442e6879bd + # via -r requirements.in +gitdb==4.0.7 \ + --hash=sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0 \ + --hash=sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005 + # via gitpython +gitpython==3.1.14 \ + --hash=sha256:3283ae2fba31c913d857e12e5ba5f9a7772bbc064ae2bb09efafa71b0dd4939b \ + --hash=sha256:be27633e7509e58391f10207cd32b2a6cf5b908f92d9cd30da2e514e1137af61 + # via -r requirements.in +gnupg==2.3.1 \ + --hash=sha256:8db5a05c369dbc231dab4c98515ce828f2dffdc14f1534441a6c59b71c6d2031 + # via -r requirements.in +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c + # via requests +iniconfig==1.1.1 \ + --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ + --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 + # via pytest +jinja2==2.10.1 \ + --hash=sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013 \ + --hash=sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b + # via -r requirements.in +markupsafe==1.1.0 \ + --hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \ + --hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \ + --hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \ + --hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \ + --hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \ + --hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \ + --hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \ + --hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \ + --hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \ + --hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \ + --hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \ + --hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \ + --hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \ + --hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \ + --hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \ + --hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \ + --hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \ + --hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \ + --hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \ + --hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \ + --hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \ + --hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \ + --hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \ + --hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \ + --hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \ + --hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \ + --hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \ + --hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 + # via jinja2 +packaging==20.9 \ + --hash=sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5 \ + --hash=sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a + # via pytest +parameters-validation==1.2.0 \ + --hash=sha256:046d512a6a9b1d55cd1f51fec72df87dbb488152a6c4a9776347313e475f31d4 \ + --hash=sha256:1db2aed4681b6c388c7334f8aa0d4f9fb373e8eca43d0f0851fb17a1db9ced1d + # via -r requirements.in +pathlib2==2.3.5 \ + --hash=sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db \ + --hash=sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868 + # via -r requirements.in +pluggy==0.13.1 \ + --hash=sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0 \ + --hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d + # via pytest +psutil==5.8.0 \ + --hash=sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64 \ + --hash=sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131 \ + --hash=sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c \ + --hash=sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6 \ + --hash=sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023 \ + --hash=sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df \ + --hash=sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394 \ + --hash=sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4 \ + --hash=sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b \ + --hash=sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2 \ + --hash=sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d \ + --hash=sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65 \ + --hash=sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d \ + --hash=sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef \ + --hash=sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7 \ + --hash=sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60 \ + --hash=sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6 \ + --hash=sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8 \ + --hash=sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b \ + --hash=sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d \ + --hash=sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac \ + --hash=sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935 \ + --hash=sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d \ + --hash=sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28 \ + --hash=sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876 \ + --hash=sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0 \ + --hash=sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3 \ + --hash=sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563 + # via gnupg +py==1.10.0 \ + --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \ + --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a + # via pytest +pycurl==7.43.0.6 \ + --hash=sha256:8301518689daefa53726b59ded6b48f33751c383cf987b0ccfbbc4ed40281325 + # via -r requirements.in +pygithub==1.54.1 \ + --hash=sha256:300bc16e62886ca6537b0830e8f516ea4bc3ef12d308e0c5aff8bdbd099173d4 \ + --hash=sha256:87afd6a67ea582aa7533afdbf41635725f13d12581faed7e3e04b1579c0c0627 + # via -r requirements.in +pyjwt==1.7.1 \ + --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \ + --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 + # via pygithub +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b + # via packaging +pytest==6.2.3 \ + --hash=sha256:671238a46e4df0f3498d1c3270e5deb9b32d25134c99b7d75370a68cfbe9b634 \ + --hash=sha256:6ad9c7bdf517a808242b998ac20063c41532a570d088d77eec1ee12b0b5574bc + # via -r requirements.in +python-string-utils==1.0.0 \ + --hash=sha256:dcf9060b03f07647c0a603408dc8b03f807f3b54a05c6e19eb14460256fac0cb \ + --hash=sha256:f1a88700baf99db1a9b6953f44181ad9ca56623c81e257e6009707e2e7851fa4 + # via -r requirements.in +pyyaml==5.3.1 \ + --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \ + --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \ + --hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \ + --hash=sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e \ + --hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \ + --hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \ + --hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \ + --hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \ + --hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \ + --hash=sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a \ + --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \ + --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \ + --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a + # via -r requirements.in +requests==2.22.0 \ + --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \ + --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 + # via + # -r requirements.in + # docker + # pygithub +six==1.14.0 \ + --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ + --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c + # via pathlib2 +smmap==4.0.0 \ + --hash=sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182 \ + --hash=sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2 + # via gitdb +toml==0.10.2 \ + --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ + --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f + # via pytest +urllib3==1.25.8 \ + --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \ + --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc + # via + # -r requirements.in + # requests +websocket-client==1.0.1 \ + --hash=sha256:3e2bf58191d4619b161389a95bdce84ce9e0b24eb8107e7e590db682c2d0ca81 \ + --hash=sha256:abf306dc6351dcef07f4d40453037e51cc5d9da2ef60d0fc5d0fe3bcda255372 + # via docker +wheel==0.36.2 \ + --hash=sha256:78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e \ + --hash=sha256:e11eefd162658ea59a60a0f6c7d493a7190ea4b9a85e335b33489d9f17e0245e + # via -r requirements.in +wrapt==1.12.1 \ + --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 + # via deprecated diff --git a/packaging_automation/tests/test.py b/packaging_automation/tests/test.py new file mode 100644 index 00000000..326092c2 --- /dev/null +++ b/packaging_automation/tests/test.py @@ -0,0 +1,3 @@ +import re + +print(re.escape("10.0")) \ No newline at end of file diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 099a34e5..a4452d5c 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -8,9 +8,10 @@ from ..common_tool_methods import ( find_nth_occurrence_position, is_major_release, - str_array_to_str, run, remove_text_with_paranthesis, get_version_details, + str_array_to_str, run, remove_text_with_parenthesis, get_version_details, replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, - get_project_version_from_tag_name, find_nth_matching_line_and_line_number) + get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, + get_patch_version_regex) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -44,7 +45,7 @@ def test_run(self): def test_remove_parentheses_from_string(self): self.assertEqual("out of parentheses ", - remove_text_with_paranthesis("out of parentheses (inside parentheses)")) + remove_text_with_parenthesis("out of parentheses (inside parentheses)")) def test_get_version_details(self): self.assertEqual({"major": "10", "minor": "0", "patch": "1"}, get_version_details("10.0.1")) @@ -80,6 +81,12 @@ def test_getprs_with_backlog_label(self): self.assertEqual(1, len(prs_backlog)) self.assertEqual(4746, prs_backlog[0].number) + def test_get_minor_version(self): + self.assertEqual("10.0", get_minor_version("10.0.3")) + + def test_get_patch_version_regex(self): + self.assertEqual("10\.0\.\d{1,3}", get_patch_version_regex("10.0.3")) + if __name__ == '__main__': unittest.main() diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index ae3dac7e..bba220c1 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -2,27 +2,28 @@ import os from datetime import datetime -from ..prepare_release import (update_release, MULTI_EXTENSION_OUT, MULTI_EXTENSION_SQL, CONFIGURE, +from ..prepare_release import (update_release, MULTI_EXTENSION_OUT, MULTI_EXTENSION_SQL, CONFIGURE, CONFIGURE_IN, CITUS_CONTROL, CONFIG_PY) from ..common_tool_methods import file_includes_line, count_line_in_file, run github_token = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[2] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") -TEST_BASE_PATH = f"{BASE_PATH}/citus" -CITUS_PROJECT_TEST_PATH = f"{TEST_BASE_PATH}/projects/citus" -CITUS_PROJECT_TEST_PATH_COPY = f"{TEST_BASE_PATH}/projects/citus_copy" + MAIN_BRANCH = "test-tools-scripts" +TEST_CHECKOUT_DIR = "citus_test" +TEST_BASE_PATH = f"{BASE_PATH}/{TEST_CHECKOUT_DIR}" def initialize_env(): - if not os.path.exists("citus"): - run("git clone https://github.com/citusdata/citus.git") + clear_env() + if not os.path.exists(TEST_CHECKOUT_DIR): + run(f"git clone https://github.com/citusdata/citus.git {TEST_CHECKOUT_DIR}") def test_major_release(): initialize_env() - os.chdir("citus") + os.chdir(TEST_CHECKOUT_DIR) try: update_release_return_value = update_release( github_token=github_token, project_name="citus", project_version="10.2.0", main_branch=MAIN_BRANCH, @@ -31,15 +32,15 @@ def test_major_release(): run(f"git checkout {update_release_return_value.release_branch_name}") - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.0") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.0])") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.0'") assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + r"\`configure' configures Citus 10.2.0 to adapt to many kinds of systems.") assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + ' short | recursive ) echo "Configuration of Citus 10.2.0:";;') + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") run(f"git checkout {update_release_return_value.upcoming_version_branch}") @@ -66,7 +67,7 @@ def test_major_release(): def test_patch_release(): initialize_env() - os.chdir("citus") + os.chdir(TEST_CHECKOUT_DIR) try: update_release(github_token=github_token, project_name="citus", project_version="10.2.0", main_branch=MAIN_BRANCH, @@ -94,6 +95,6 @@ def test_patch_release(): def clear_env(): - if os.path.exists("../citus"): + if os.path.exists(f"../{TEST_CHECKOUT_DIR}"): os.chdir("..") - run("sudo rm -rf citus") + run(f"sudo rm -rf {TEST_CHECKOUT_DIR}") From 48ff45c363b94d66c83d73659fbeff463c4f2ece Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 27 May 2021 14:06:28 +0300 Subject: [PATCH 075/106] Remove unnecessary test file --- packaging_automation/tests/test.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 packaging_automation/tests/test.py diff --git a/packaging_automation/tests/test.py b/packaging_automation/tests/test.py deleted file mode 100644 index 326092c2..00000000 --- a/packaging_automation/tests/test.py +++ /dev/null @@ -1,3 +0,0 @@ -import re - -print(re.escape("10.0")) \ No newline at end of file From d75a8cfc4fcfc9909df67b3e8cca1673c1e5758e Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 27 May 2021 14:24:49 +0300 Subject: [PATCH 076/106] Fix missing method import problem for prepare release tests --- packaging_automation/prepare_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index b8043519..f421f5b6 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -10,7 +10,7 @@ from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, - find_nth_matching_line_and_line_number) + find_nth_matching_line_and_line_number,get_minor_version) from .common_validations import CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" From e62e3bc22d4744b0c2462928b342256588e969f2 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 27 May 2021 23:11:29 +0300 Subject: [PATCH 077/106] Execute unit tests and fix bugs --- packaging_automation/common_tool_methods.py | 4 ++-- packaging_automation/prepare_release.py | 23 ++++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index c9d365ce..7f4e57a8 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -78,7 +78,7 @@ def get_minor_version(version: str) -> str: def get_patch_version_regex(version: is_version(str)): - return f"{re.escape(get_minor_version(version))}{PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX}" + return fr"^{re.escape(get_minor_version(version))}{PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX}$" def is_merge_commit(commit: Commit): @@ -161,7 +161,7 @@ def replace_line_in_file(file: str, match_regex: str, replace_str: str) -> bool: lines = file_content.splitlines() has_match = False for line_number, line in enumerate(lines): - if re.match(match_regex, line): + if re.match(match_regex, line.strip()): has_match = True lines[line_number] = replace_str edited_content = str_array_to_str(lines) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index f421f5b6..99f36b09 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -10,7 +10,7 @@ from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, - find_nth_matching_line_and_line_number,get_minor_version) + find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex) from .common_validations import CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -26,7 +26,6 @@ MULTI_EXT_PATCH_SEARCH_PATTERN = r"^\s*" + CITUS_PATCH_VERSION_PATTERN + r"$" CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN = r"^MASTER_VERSION = '\d+\.\d+'" - CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" REPO_OWNER = "citusdata" @@ -86,9 +85,6 @@ class PatchReleaseParams: BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] - - - def update_release(github_token: str, project_name: str, project_version: is_version(str), main_branch: str, earliest_pr_date: datetime, exec_path: str, is_test: bool = False, cherry_pick_enabled: bool = False) -> UpdateReleaseReturnValue: @@ -161,8 +157,8 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar # execute "auto-conf " execute_autoconf_f() # change version info in multi_extension.out - update_version_in_multi_extension_out(patchReleaseParams.multi_extension_out_path, - patchReleaseParams.project_version) + update_version_in_multi_extension_out_for_patch(patchReleaseParams.multi_extension_out_path, + patchReleaseParams.project_version) if patchReleaseParams.cherry_pick_enabled: # cherry-pick the pr's with backport labels cherrypick_prs_with_backport_labels(patchReleaseParams.earliest_pr_date, patchReleaseParams.main_branch, @@ -337,8 +333,19 @@ def update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_m def update_version_in_multi_extension_out(multi_extension_out_path, project_version): print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...###") + + if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DEVEL_SEARCH_PATTERN, + f" {project_version}"): + raise ValueError( + f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") + print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.###") + + +def update_version_in_multi_extension_out_for_patch(multi_extension_out_path, project_version): + print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...###") + if not replace_line_in_file(multi_extension_out_path, - f"{re.escape(get_minor_version(project_version))}{PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX}", + get_patch_version_regex(project_version), f" {project_version}"): raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") From 34b2514fa8223a22d1a0305a8ed7c27c50ef5b09 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 28 May 2021 10:04:55 +0300 Subject: [PATCH 078/106] ADd missing MAIN_BRANCH variable to fix build --- .github/workflows/tool-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 24816967..c0a9f96b 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -4,6 +4,7 @@ env: GH_TOKEN: ${{ secrets.GH_TOKEN }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay + MAIN_BRANCH: all-citus on: push: From 7e72c1f8c377a2f86f6bbce21ea881f7617bd123 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 28 May 2021 10:28:21 +0300 Subject: [PATCH 079/106] Fix formatting of python command --- bash/update_package_properties.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bash/update_package_properties.sh b/bash/update_package_properties.sh index 4998a3f6..2c947d3b 100755 --- a/bash/update_package_properties.sh +++ b/bash/update_package_properties.sh @@ -26,8 +26,15 @@ commit_message="Bump to ${PRJ_NAME} ${PRJ_VER}" git checkout -b "${pr_branch_name}" -python tools/python/update_package_properties.py --gh_token "${GH_TOKEN}" --prj_name "${PRJ_NAME}" --tag_name "${TAG_NAME}" --fancy "${FANCY}" \ - --fancy_ver_no "${FANCY_VERSION_NO}" --email "${MICROSOFT_EMAIL}" --name "${NAME}" --date "$(date '+%Y.%m.%d %H:%M:%S %z')" --exec_path "$(pwd)" +python tools/python/update_package_properties.py --gh_token "${GH_TOKEN}" \ + --prj_name "${PRJ_NAME}" \ + --tag_name "${TAG_NAME}" \ + --fancy "${FANCY}" \ + --fancy_ver_no "${FANCY_VERSION_NO}" \ + --email "${MICROSOFT_EMAIL}" \ + --name "${NAME}" \ + --date "$(date '+%Y.%m.%d %H:%M:%S %z')" \ + --exec_path "$(pwd)" git commit -a -m "${commit_message}" From 68602486ac7fe4cab5acebf3fdf562cfedf9a3e5 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 28 May 2021 15:54:04 +0300 Subject: [PATCH 080/106] Fix round 3 of comments of @hanefi and @onur --- packaging_automation/common_tool_methods.py | 27 +++++++++++++++++- packaging_automation/prepare_release.py | 12 ++++---- .../tests/test_common_tool_methods.py | 28 ++++++++++++++++--- .../tests/test_prepare_release.py | 7 +++-- .../update_package_properties.py | 8 +++--- 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 7f4e57a8..a2d36807 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -53,7 +53,7 @@ def find_nth_matching_line_and_line_number(subject_string: str, regex_pattern: s def remove_text_with_parenthesis(param: str) -> str: - """Removes texts within parentheses i.e. outside parenthesis(inside parenthesis)-> outside parenthesis """ + """Removes texts within parenthesis i.e. outside parenthesis(inside parenthesis)-> outside parenthesis """ return re.sub(r"[(\[].*?[)\]]", "", param) @@ -171,6 +171,31 @@ def replace_line_in_file(file: str, match_regex: str, replace_str: str) -> bool: return has_match +def append_line_in_file(file: str, match_regex: str, append_str: str) -> bool: + with open(file, "r+") as reader: + file_content = reader.read() + lines = file_content.splitlines() + has_match = False + appended_lines = lines.copy() + appended_line_index = 0 + for line_number, line in enumerate(lines): + if re.match(match_regex, line.strip()): + has_match = True + + if line_number + 1 < len(lines): + appended_lines[appended_line_index + 1] = append_str + lines_to_be_shifted = lines[line_number + 1:] + appended_lines.extend(lines_to_be_shifted) + else: + appended_lines.append(append_str) + appended_line_index = appended_line_index + 1 + edited_content = str_array_to_str(appended_lines) + with open(file, "w") as writer: + writer.write(edited_content) + + return has_match + + def get_current_branch() -> str: repo = Repo(BASE_GIT_PATH) return repo.active_branch diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 99f36b09..473fb862 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -10,7 +10,7 @@ from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, - find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex) + find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex,append_line_in_file) from .common_validations import CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -22,8 +22,8 @@ CONFIGURE = "configure" CITUS_CONTROL_SEARCH_PATTERN = r"^default_version*" -MULTI_EXT_DEVEL_SEARCH_PATTERN = r"^\s*" + CITUS_MINOR_VERSION_PATTERN + "devel$" -MULTI_EXT_PATCH_SEARCH_PATTERN = r"^\s*" + CITUS_PATCH_VERSION_PATTERN + r"$" +MULTI_EXT_DEVEL_SEARCH_PATTERN = rf"^\s*{CITUS_MINOR_VERSION_PATTERN}devel$" +MULTI_EXT_PATCH_SEARCH_PATTERN = rf"^\s*{CITUS_PATCH_VERSION_PATTERN}$" CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN = r"^MASTER_VERSION = '\d+\.\d+'" CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" @@ -109,7 +109,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver repository = g.get_repo(f"{REPO_OWNER}/{project_name}") newly_created_sql_file = "" - # if major release + # major release if is_major_release(project_version): print(f"### {project_version} is a major release. Executing Major release flow###") major_release_params = MajorReleaseParams(configure_in_path=configure_in_path, devel_version=devel_version, @@ -135,6 +135,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver newly_created_sql_file = prepare_upcoming_version_branch(branch_params) print(f"OK {project_version} Major release flow executed successfully") + # patch release else: patch_release_params = PatchReleaseParams(cherry_pick_enabled=cherry_pick_enabled, configure_in_path=configure_in_path, @@ -291,7 +292,7 @@ def update_schema_version_with_upcoming_version_in_multi_extension_file(current_ f"### Updating schema version {current_schema_version} on {multi_extension_sql_path} " f"file with the upcoming version {upcoming_version}...### ") # TODO Append instead of replace may require - if not replace_line_in_file(multi_extension_sql_path, + if not append_line_in_file(multi_extension_sql_path, f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", f"ALTER EXTENSION citus UPDATE TO '{upcoming_minor_version}-1';"): raise ValueError(f"{multi_extension_sql_path} does not have match for version") @@ -389,6 +390,5 @@ def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path content = content + "\n\n" content = content + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" f_writer.write(content) - # run(f"git add {distributed_dir_path}/{newly_created_sql_file}") print(f"### OK {newly_created_sql_file} created.") return newly_created_sql_file diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index a4452d5c..bce92426 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -11,7 +11,7 @@ str_array_to_str, run, remove_text_with_parenthesis, get_version_details, replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, - get_patch_version_regex) + get_patch_version_regex, append_line_in_file) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -43,9 +43,9 @@ def test_run(self): result = run("echo 'Run' method is performing fine ") self.assertEqual(0, result.returncode) - def test_remove_parentheses_from_string(self): - self.assertEqual("out of parentheses ", - remove_text_with_parenthesis("out of parentheses (inside parentheses)")) + def test_remove_paranthesis_from_string(self): + self.assertEqual("out of paranthesis ", + remove_text_with_parenthesis("out of paranthesis (inside paranthesis)")) def test_get_version_details(self): self.assertEqual({"major": "10", "minor": "0", "patch": "1"}, get_version_details("10.0.1")) @@ -87,6 +87,26 @@ def test_get_minor_version(self): def test_get_patch_version_regex(self): self.assertEqual("10\.0\.\d{1,3}", get_patch_version_regex("10.0.3")) + def test_append_line_in_file(self): + try: + with open("test_append.txt", "a") as writer: + writer.write("Test line 1\n") + writer.write("Test line 2\n") + append_line_in_file("test_append.txt", "^Test line 1", "Test line 1.5") + append_line_in_file("test_append.txt", "^Test line 2", "Test line 2.5") + + with open("test_append.txt", "r") as reader: + line1 = reader.readline() + line2 = reader.readline() + line3 = reader.readline() + line4 = reader.readline() + self.assertEqual(line1, "Test line 1\n") + self.assertEqual(line2, "Test line 1.5\n") + self.assertEqual(line3, "Test line 2\n") + self.assertEqual(line4, "Test line 2.5\n") + finally: + os.remove("test_append.txt") + if __name__ == '__main__': unittest.main() diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index bba220c1..12bd48a5 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -45,9 +45,13 @@ def test_major_release(): run(f"git checkout {update_release_return_value.upcoming_version_branch}") assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.1-1';") assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, "ALTER EXTENSION citus UPDATE TO '10.2-1';") assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_SQL, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 assert file_includes_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") @@ -96,5 +100,4 @@ def test_patch_release(): def clear_env(): if os.path.exists(f"../{TEST_CHECKOUT_DIR}"): - os.chdir("..") - run(f"sudo rm -rf {TEST_CHECKOUT_DIR}") + run(f"sudo rm -rf ../{TEST_CHECKOUT_DIR}") diff --git a/packaging_automation/update_package_properties.py b/packaging_automation/update_package_properties.py index 345904b0..83285e64 100644 --- a/packaging_automation/update_package_properties.py +++ b/packaging_automation/update_package_properties.py @@ -123,7 +123,7 @@ def get_last_changelog_content_from_debian(all_changelog_content: str) -> str: return changelogs -def remove_parentheses_from_string(param: str) -> str: +def remove_paranthesis_from_string(param: str) -> str: return re.sub(r"[(\[].*?[)\]]", "", param) @@ -135,15 +135,15 @@ def changelog_for_tag(github_token: str, project_name: str, tag_name: str) -> st return last_changelog_content -# truncates # chars , get the version an put parentheses around version number adds 'stable; urgency=low' at the end +# truncates # chars , get the version an put paranthesis around version number adds 'stable; urgency=low' at the end # changelog_header=> ### citus v8.3.3 (March 23, 2021) ### # debian header => citus (10.0.3.citus-1) stable; urgency=low @validate_parameters def debian_changelog_header(changelog_header: is_project_changelog_header(str), fancy: bool, fancy_version_number: int) -> str: hash_removed_string = changelog_header.lstrip("### ").rstrip(" ###") - parentheses_removed_string = remove_parentheses_from_string(hash_removed_string) - words = parentheses_removed_string.strip().split(" ") + paranthesis_removed_string = remove_paranthesis_from_string(hash_removed_string) + words = paranthesis_removed_string.strip().split(" ") if len(words) != 2: raise ValueError("Two words should be included in striped version header") project_name = words[0] From 901ea18e4befd54c30c2a5133c423adb5232fc74 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 28 May 2021 17:14:40 +0300 Subject: [PATCH 081/106] Fix test execution problem --- packaging_automation/tests/test_prepare_release.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 12bd48a5..d6a17c30 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -70,6 +70,7 @@ def test_major_release(): def test_patch_release(): + print(os.getcwd()) initialize_env() os.chdir(TEST_CHECKOUT_DIR) try: @@ -100,4 +101,5 @@ def test_patch_release(): def clear_env(): if os.path.exists(f"../{TEST_CHECKOUT_DIR}"): - run(f"sudo rm -rf ../{TEST_CHECKOUT_DIR}") + os.chdir("..") + run(f"sudo rm -rf {TEST_CHECKOUT_DIR}") From fa1005ad228e5cbd00ec49bf0bc5302b2922e7f3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 28 May 2021 22:51:35 +0300 Subject: [PATCH 082/106] Format log messages --- packaging_automation/prepare_release.py | 90 +++++++++++++------------ 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 473fb862..a15109c9 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -10,7 +10,8 @@ from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, - find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex,append_line_in_file) + find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, + append_line_in_file) from .common_validations import CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -111,7 +112,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver # major release if is_major_release(project_version): - print(f"### {project_version} is a major release. Executing Major release flow###") + print(f"### {project_version} is a major release. Executing Major release flow... ###") major_release_params = MajorReleaseParams(configure_in_path=configure_in_path, devel_version=devel_version, is_test=is_test, main_branch=main_branch, multi_extension_out_path=multi_extension_out_path, @@ -134,7 +135,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver multi_extension_sql_path=multi_extension_sql_path) newly_created_sql_file = prepare_upcoming_version_branch(branch_params) - print(f"OK {project_version} Major release flow executed successfully") + print(f"### Done {project_version} Major release flow executed successfully. ###") # patch release else: patch_release_params = PatchReleaseParams(cherry_pick_enabled=cherry_pick_enabled, @@ -150,7 +151,7 @@ def update_release(github_token: str, project_name: str, project_version: is_ver def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleaseParams): - print(f"### {patchReleaseParams.project_version} is a patch release. Executing Patch release flow ###") + print(f"### {patchReleaseParams.project_version} is a patch release. Executing Patch release flow... ###") # checkout release branch (release-X.Y) checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) # change version info in configure.in file @@ -172,9 +173,11 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar if not patchReleaseParams.is_test: push_branch(release_pr_branch) + print(f"### Done Patch release flow executed successfully. ###") + def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams): - print(f"### Preparing {upcoming_params.upcoming_version_branch} branch that bumps master version.###") + print(f"### {upcoming_params.upcoming_version_branch} flow is being executed... ###") # checkout master checkout_branch(upcoming_params.main_branch, upcoming_params.is_test) # create master-update-version-$curtime branch @@ -216,12 +219,12 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams create_pull_request_for_upcoming_version_branch(upcoming_params.repository, upcoming_params.main_branch, upcoming_params.upcoming_version_branch, upcoming_params.upcoming_version) - print(f"### OK {upcoming_params.upcoming_version_branch} prepared.###") + print(f"### Done {upcoming_params.upcoming_version_branch} flow executed. ###") return newly_created_sql_file def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleaseParams): - print(f"### Preparing {majorReleaseParams.release_branch_name}...###") + print(f"### {majorReleaseParams.release_branch_name} release branch flow is being executed... ###") # checkout master checkout_branch(majorReleaseParams.main_branch, majorReleaseParams.is_test) # create release branch in release-X.Y format @@ -238,69 +241,68 @@ def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleasePar # push release branch (No PR creation!!!) if not majorReleaseParams.is_test: push_branch(majorReleaseParams.release_branch_name) - print(f"### OK {majorReleaseParams.release_branch_name} prepared.###") + print(f"### Done {majorReleaseParams.release_branch_name} release branch flow executed .###") def cherrypick_prs_with_backport_labels(earliest_pr_date, main_branch, release_branch_name, repository): print( - f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}### ") + f"### Getting all PR with backport label after {datetime.strftime(earliest_pr_date, '%Y.%m.%d %H:%M')}... ### ") prs_with_earliest_date = get_prs_for_patch_release(repository, earliest_pr_date, main_branch) # get commits for selected prs with backport label prs_with_backport = filter_prs_by_label(prs_with_earliest_date, "backport") - print(f"### OK {len(prs_with_backport)} PRs with backport label found. PR list is as below###") + print(f"### Done {len(prs_with_backport)} PRs with backport label found. PR list is as below. ###") for pr in prs_with_backport: print(f"\tNo:{pr.number} Title:{pr.title}") # cherrypick all commits with backport label - print(f"Cherry-picking PRs to {release_branch_name}...") + print(f"### Cherry-picking PRs to {release_branch_name}... ###") cherry_pick_prs(prs_with_backport) - print(f"OK Cherry pick completed for all PRs on branch {release_branch_name}") + print(f"### Done Cherry pick completed for all PRs on branch {release_branch_name}. ###") def create_pull_request_for_upcoming_version_branch(repository, main_branch, upcoming_version_branch, upcoming_version): - print(f"### Creating pull request for {upcoming_version_branch}###") + print(f"### Creating pull request for {upcoming_version_branch}... ###") pr_result = repository.create_pull(title=f"Bump Citus to {upcoming_version}", base=main_branch, head=upcoming_version_branch, body="") - print(f"### OK Pull request created. PR Number:{pr_result.number} PR URL: {pr_result.url}###") + print(f"### Done Pull request created. PR no:{pr_result.number} PR URL: {pr_result.url}. ### ") def push_branch(upcoming_version_branch): - print(f"Pushing changes for {upcoming_version_branch} into remote origin...###") + print(f"Pushing changes for {upcoming_version_branch} into remote origin... ###") run(f"git push --set-upstream origin {upcoming_version_branch}") - print(f"### OK Changes pushed for {upcoming_version_branch}###") + print(f"### Done Changes pushed for {upcoming_version_branch}. ###") def commit_changes_for_version_bump(project_name, project_version): current_branch = get_current_branch() - print(f"### Committing changes for branch {current_branch}...###") + print(f"### Committing changes for branch {current_branch}... ###") run(f' git commit -a -m "Bump {project_name} version to {project_version} "') - print(f"### OK Changes committed for {current_branch}###") + print(f"### Done Changes committed for {current_branch}. ###") def update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version): - print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}...###") + print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}... ###") if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, f"default_version = '{upcoming_minor_version}-1'"): raise ValueError(f"{citus_control_file_path} does not have match for version") - print(f"### OK {citus_control_file_path} file is updated with the upcoming version {upcoming_minor_version}...###") + print(f"### Done {citus_control_file_path} file is updated with the upcoming version {upcoming_minor_version}. ###") def update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, multi_extension_sql_path, upcoming_minor_version, upcoming_version): - print( - f"### Updating schema version {current_schema_version} on {multi_extension_sql_path} " - f"file with the upcoming version {upcoming_version}...### ") + print(f"### Updating schema version {current_schema_version} on {multi_extension_sql_path} " + f"file with the upcoming version {upcoming_version}... ### ") # TODO Append instead of replace may require if not append_line_in_file(multi_extension_sql_path, - f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", - f"ALTER EXTENSION citus UPDATE TO '{upcoming_minor_version}-1';"): + f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", + f"ALTER EXTENSION citus UPDATE TO '{upcoming_minor_version}-1';"): raise ValueError(f"{multi_extension_sql_path} does not have match for version") - print(f"### OK Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}###") + print(f"### Done Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}. ###") def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: - print(f"### Reading current schema version from {citus_control_file_path}...###") + print(f"### Reading current schema version from {citus_control_file_path}... ###") current_schema_version = "" with open(citus_control_file_path, "r") as cc_reader: cc_file_content = cc_reader.read() @@ -320,75 +322,75 @@ def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: raise ValueError("Version info could not be found in citus.control file") current_schema_version = current_schema_version.strip(" '") - print(f"OK Schema version is {current_schema_version}") + print(f"### Done Schema version is {current_schema_version}. ###") return current_schema_version def update_version_with_upcoming_version_in_config_py(config_py_path, upcoming_minor_version): - print(f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}...###") + print(f"### Updating {config_py_path} file with the upcoming version {upcoming_minor_version}... ###") if not replace_line_in_file(config_py_path, CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN, f"MASTER_VERSION = '{upcoming_minor_version}'"): raise ValueError(f"{config_py_path} does not have match for version") - print(f"### {config_py_path} file updated with the upcoming version {upcoming_minor_version}...###") + print(f"### Done {config_py_path} file updated with the upcoming version {upcoming_minor_version}. ###") def update_version_in_multi_extension_out(multi_extension_out_path, project_version): - print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...###") + print(f"### Updating {multi_extension_out_path} file with the project version {project_version}... ###") if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DEVEL_SEARCH_PATTERN, f" {project_version}"): raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") - print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.###") + print(f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") def update_version_in_multi_extension_out_for_patch(multi_extension_out_path, project_version): - print(f"### Updating {multi_extension_out_path} file with the project version {project_version}...###") + print(f"### Updating {multi_extension_out_path} file with the project version {project_version}... ###") if not replace_line_in_file(multi_extension_out_path, get_patch_version_regex(project_version), f" {project_version}"): raise ValueError( f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") - print(f"### OK {multi_extension_out_path} file is updated with project version {project_version}.###") + print(f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") def execute_autoconf_f(): - print(f"### Executing autoconf -f command...###") + print(f"### Executing autoconf -f command... ###") run("autoconf -f") - print(f"### OK autoconf -f executed.###") + print(f"### Done autoconf -f executed. ###") def update_version_in_configure_in(configure_in_path, project_version): - print(f"### Updating version on file {configure_in_path}...###") + print(f"### Updating version on file {configure_in_path}... ###") if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, f"AC_INIT([Citus], [{project_version}])"): raise ValueError(f"{configure_in_path} does not have match for version") - print(f"### OK {configure_in_path} file is updated with project version {project_version}.###") + print(f"### Done {configure_in_path} file is updated with project version {project_version}. ###") def create_and_checkout_branch(release_branch_name): - print(f"### Creating release branch with name {release_branch_name} from {get_current_branch()}...###") + print(f"### Creating release branch with name {release_branch_name} from {get_current_branch()}... ###") run(f'git checkout -b {release_branch_name}') - print(f"### OK {release_branch_name} created###") + print(f"### Done {release_branch_name} created. ###") def checkout_branch(branch_name, is_test): - print(f"### Checking out {branch_name}...###") + print(f"### Checking out {branch_name}... ###") run(f"git checkout {branch_name}") if not is_test: run(f"git pull") - print(f"### OK {branch_name} checked out and pulled###") + print(f"### Done {branch_name} checked out and pulled. ###") def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path, upcoming_minor_version): newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" - print(f"### Creating file {newly_created_sql_file}...###") + print(f"### Creating file {newly_created_sql_file}... ###") with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: content = f"/* citus--{current_schema_version}--{upcoming_minor_version}-1 */" content = content + "\n\n" content = content + f"-- bump version to {upcoming_minor_version}-1" + "\n\n" f_writer.write(content) - print(f"### OK {newly_created_sql_file} created.") + print(f"### Done {newly_created_sql_file} created. ###") return newly_created_sql_file From 9193cf655dca085f20d56469aa867ef17141cd80 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 31 May 2021 14:10:14 +0300 Subject: [PATCH 083/106] Add init function for prepare_release to execute on bash wit parameters --- packaging_automation/prepare_release.py | 29 +++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index a15109c9..6c7c4c7e 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -1,18 +1,19 @@ import os -import re import uuid from dataclasses import dataclass from datetime import datetime +import argparse import pathlib2 from github import Github, Repository +from parameters_validation import (non_blank, non_empty) from .common_tool_methods import (get_version_details, get_upcoming_patch_version, is_major_release, get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, append_line_in_file) -from .common_validations import CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version +from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" CITUS_CONTROL = "src/backend/distributed/citus.control" @@ -86,8 +87,9 @@ class PatchReleaseParams: BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] -def update_release(github_token: str, project_name: str, project_version: is_version(str), main_branch: str, - earliest_pr_date: datetime, exec_path: str, is_test: bool = False, +def update_release(github_token: non_blank(non_empty(str)), project_name: non_blank(non_empty(str)), + project_version: is_version(str), main_branch: non_blank(non_empty(str)), + earliest_pr_date: datetime, exec_path: non_blank(non_empty(str)), is_test: bool = False, cherry_pick_enabled: bool = False) -> UpdateReleaseReturnValue: multi_extension_sql_path = f"{exec_path}/{MULTI_EXTENSION_SQL}" citus_control_file_path = f"{exec_path}/{CITUS_CONTROL}" @@ -394,3 +396,22 @@ def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path f_writer.write(content) print(f"### Done {newly_created_sql_file} created. ###") return newly_created_sql_file + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--gh_token') + parser.add_argument('--prj_name') + parser.add_argument('--prj_ver') + parser.add_argument('--main_branch') + parser.add_argument('--earliest_pr_date') + parser.add_argument('--exec_path') + parser.add_argument('--cherry_pick_enabled', nargs='?', default="True") + parser.add_argument('--is_test', nargs='?', default="False") + arguments = parser.parse_args() + + update_release(github_token=arguments.gh_token, project_name=arguments.prj_name, project_version=arguments.prj_ver, + main_branch=arguments.main_branch, + earliest_pr_date=datetime.strptime(arguments.earliest_pr_date, '%Y.%m.%d %H:%M:%S %z'), + is_test=arguments.is_test.lower() == "true", + cherry_pick_enabled=arguments.cherry_pick_enabled.lower() == "true", exec_path=arguments.exec_path) From cd390b630b6bec6e3c362ba09a4f9d4ea36ef2cd Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 1 Jun 2021 16:16:04 +0300 Subject: [PATCH 084/106] Add extension updates on multi_extension files Add prepend option in common_tools --- packaging_automation/common_tool_methods.py | 37 ++++- packaging_automation/prepare_release.py | 143 +++++++++++++----- .../multi_extension_out_prepare_release.tmpl | 17 +++ .../multi_extension_sql_prepare_release.tmpl | 9 ++ .../tests/test_common_tool_methods.py | 62 ++++++-- .../tests/test_prepare_release.py | 19 ++- .../update_package_properties.py | 13 +- 7 files changed, 232 insertions(+), 68 deletions(-) create mode 100644 packaging_automation/templates/multi_extension_out_prepare_release.tmpl create mode 100644 packaging_automation/templates/multi_extension_sql_prepare_release.tmpl diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index a2d36807..e9f57586 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -176,20 +176,41 @@ def append_line_in_file(file: str, match_regex: str, append_str: str) -> bool: file_content = reader.read() lines = file_content.splitlines() has_match = False - appended_lines = lines.copy() + copy_lines = lines.copy() appended_line_index = 0 for line_number, line in enumerate(lines): if re.match(match_regex, line.strip()): has_match = True if line_number + 1 < len(lines): - appended_lines[appended_line_index + 1] = append_str + copy_lines[appended_line_index + 1] = append_str lines_to_be_shifted = lines[line_number + 1:] - appended_lines.extend(lines_to_be_shifted) + copy_lines = copy_lines[0:appended_line_index + 2] + lines_to_be_shifted else: - appended_lines.append(append_str) + copy_lines.append(append_str) appended_line_index = appended_line_index + 1 - edited_content = str_array_to_str(appended_lines) + edited_content = str_array_to_str(copy_lines) + with open(file, "w") as writer: + writer.write(edited_content) + + return has_match + + +def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: + with open(file, "r+") as reader: + file_content = reader.read() + lines = file_content.splitlines() + has_match = False + copy_lines = lines.copy() + prepended_line_index = 0 + for line_number, line in enumerate(lines): + if re.match(match_regex, line.strip()): + has_match = True + copy_lines[prepended_line_index] = append_str + lines_to_be_shifted = lines[line_number:] + copy_lines = copy_lines[0:prepended_line_index + 1] + lines_to_be_shifted + prepended_line_index = prepended_line_index + 1 + edited_content = str_array_to_str(copy_lines) with open(file, "w") as writer: writer.write(edited_content) @@ -199,3 +220,9 @@ def append_line_in_file(file: str, match_regex: str, append_str: str) -> bool: def get_current_branch() -> str: repo = Repo(BASE_GIT_PATH) return repo.active_branch + + +def get_template_environment(template_dir: str) -> Environment: + file_loader = FileSystemLoader(template_dir) + env = Environment(loader=file_loader) + return env diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 6c7c4c7e..06151b15 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -12,31 +12,48 @@ get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, - append_line_in_file) + append_line_in_file, prepend_line_in_file, get_template_environment) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" CITUS_CONTROL = "src/backend/distributed/citus.control" MULTI_EXTENSION_OUT = "src/test/regress/expected/multi_extension.out" CONFIG_PY = "src/test/regress/upgrade/config.py" -DISTRIBUTED_DIR_PATH = "src/backend/distributed" +DISTRIBUTED_SQL_DIR_PATH = "src/backend/distributed/sql" +DOWNGRADES_DIR_PATH = f"{DISTRIBUTED_SQL_DIR_PATH}/downgrades" CONFIGURE_IN = "configure.in" CONFIGURE = "configure" CITUS_CONTROL_SEARCH_PATTERN = r"^default_version*" MULTI_EXT_DEVEL_SEARCH_PATTERN = rf"^\s*{CITUS_MINOR_VERSION_PATTERN}devel$" MULTI_EXT_PATCH_SEARCH_PATTERN = rf"^\s*{CITUS_PATCH_VERSION_PATTERN}$" + +MULTI_EXT_DETAIL_PREFIX = rf"DETAIL: Loaded library requires " +MULTI_EXT_DETAIL1_SUFFIX = rf", but 8.0-1 was specified." +MULTI_EXT_DETAIL2_SUFFIX = rf", but the installed extension version is 8.1-1." +MULTI_EXT_DETAIL1_PATTERN = rf"^{MULTI_EXT_DETAIL_PREFIX}\d+\.\d+{MULTI_EXT_DETAIL1_SUFFIX}$" + +MULTI_EXT_DETAIL2_PATTERN = ( + rf"^{MULTI_EXT_DETAIL_PREFIX}\d+\.\d+{MULTI_EXT_DETAIL2_SUFFIX}$") + CONFIG_PY_MASTER_VERSION_SEARCH_PATTERN = r"^MASTER_VERSION = '\d+\.\d+'" CONFIGURE_IN_SEARCH_PATTERN = "AC_INIT*" REPO_OWNER = "citusdata" +BASE_PATH = pathlib2.Path(__file__).parent.absolute() +TEMPLATES_PATH = f"{BASE_PATH}/templates" + +MULTI_EXT_OUT_TEMPLATE_FILE = "multi_extension_out_prepare_release.tmpl" +MULTI_EXT_SQL_TEMPLATE_FILE = "multi_extension_sql_prepare_release.tmpl" + @dataclass class UpdateReleaseReturnValue: release_branch_name: str upcoming_version_branch: str upgrade_path_sql_file: str + downgrade_path_sql_file: str @dataclass @@ -58,6 +75,7 @@ class UpcomingVersionBranchParams: configure_in_path: str devel_version: str distributed_dir_path: str + downgrades_dir_path: str is_test: bool main_branch: str multi_extension_out_path: str @@ -96,7 +114,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl multi_extension_out_path = f"{exec_path}/{MULTI_EXTENSION_OUT}" configure_in_path = f"{exec_path}/{CONFIGURE_IN}" config_py_path = f"{exec_path}/{CONFIG_PY}" - distributed_dir_path = f"{exec_path}/{DISTRIBUTED_DIR_PATH}" + distributed_dir_path = f"{exec_path}/{DISTRIBUTED_SQL_DIR_PATH}" + downgrades_dir_path = f"{exec_path}/{DOWNGRADES_DIR_PATH}" project_version_details = get_version_details(project_version) default_upcoming_version = get_upcoming_patch_version(project_version) @@ -110,7 +129,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl g = Github(github_token) repository = g.get_repo(f"{REPO_OWNER}/{project_name}") - newly_created_sql_file = "" + upgrade_file = "" + downgrade_file = "" # major release if is_major_release(project_version): @@ -131,12 +151,13 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl config_py_path=config_py_path, configure_in_path=configure_in_path, distributed_dir_path=distributed_dir_path, + downgrades_dir_path=downgrades_dir_path, repository=repository, upcoming_minor_version=upcoming_minor_version, multi_extension_out_path=multi_extension_out_path, multi_extension_sql_path=multi_extension_sql_path) - newly_created_sql_file = prepare_upcoming_version_branch(branch_params) + upgrade_file, downgrade_file = prepare_upcoming_version_branch(branch_params) print(f"### Done {project_version} Major release flow executed successfully. ###") # patch release else: @@ -149,7 +170,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl release_branch_name=release_branch_name, repository=repository) prepare_release_branch_for_patch_release(patch_release_params) return UpdateReleaseReturnValue(release_branch_name, upcoming_version_branch, - f"{DISTRIBUTED_DIR_PATH}/{newly_created_sql_file}") + f"{DISTRIBUTED_SQL_DIR_PATH}/{upgrade_file}", + f"{DOWNGRADES_DIR_PATH}/{downgrade_file}") def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleaseParams): @@ -193,22 +215,28 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams execute_autoconf_f() # update version info with upcoming version on multiextension.out update_version_in_multi_extension_out(upcoming_params.multi_extension_out_path, upcoming_params.devel_version) + # update detail lines with minor version + update_detail_strings_in_multi_extension_out(upcoming_params.multi_extension_out_path, + upcoming_params.upcoming_minor_version) # get current schema version from citus.control current_schema_version = get_current_schema_from_citus_control(upcoming_params.citus_control_file_path) - # find current schema version info and update it with upcoming version in multi_extension.sql file - update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, - upcoming_params.multi_extension_sql_path, - upcoming_params.upcoming_minor_version, - upcoming_params.upcoming_version) - # find current schema version info and update it with upcoming version in multi_extension.out file - update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, - upcoming_params.multi_extension_out_path, - upcoming_params.upcoming_minor_version, - upcoming_params.upcoming_version) + # add downgrade script in multi_extension.sql file + add_downgrade_script_in_multi_extension_file(current_schema_version, + upcoming_params.multi_extension_sql_path, + upcoming_params.upcoming_minor_version, MULTI_EXT_SQL_TEMPLATE_FILE) + # add downgrade script in multi_extension.out file + add_downgrade_script_in_multi_extension_file(current_schema_version, + upcoming_params.multi_extension_out_path, + upcoming_params.upcoming_minor_version, MULTI_EXT_OUT_TEMPLATE_FILE) # create a new sql file for upgrade path: - newly_created_sql_file = create_new_sql_for_upgrade_path(current_schema_version, - upcoming_params.distributed_dir_path, - upcoming_params.upcoming_minor_version) + upgrade_file = create_new_sql_for_upgrade_path(current_schema_version, + upcoming_params.distributed_dir_path, + upcoming_params.upcoming_minor_version) + # create a new sql file for downgrade path: + downgrade_file = create_new_sql_for_downgrade_path(current_schema_version, + upcoming_params.downgrades_dir_path, + upcoming_params.upcoming_minor_version) + # change version in citus.control file update_version_with_upcoming_version_in_citus_control(upcoming_params.citus_control_file_path, upcoming_params.upcoming_minor_version) @@ -222,7 +250,7 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams upcoming_params.upcoming_version_branch, upcoming_params.upcoming_version) print(f"### Done {upcoming_params.upcoming_version_branch} flow executed. ###") - return newly_created_sql_file + return upgrade_file, downgrade_file def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleaseParams): @@ -277,8 +305,8 @@ def push_branch(upcoming_version_branch): def commit_changes_for_version_bump(project_name, project_version): current_branch = get_current_branch() print(f"### Committing changes for branch {current_branch}... ###") - - run(f' git commit -a -m "Bump {project_name} version to {project_version} "') + run("git add .") + run(f' git commit -m "Bump {project_name} version to {project_version} "') print(f"### Done Changes committed for {current_branch}. ###") @@ -290,17 +318,23 @@ def update_version_with_upcoming_version_in_citus_control(citus_control_file_pat print(f"### Done {citus_control_file_path} file is updated with the upcoming version {upcoming_minor_version}. ###") -def update_schema_version_with_upcoming_version_in_multi_extension_file(current_schema_version, - multi_extension_sql_path, - upcoming_minor_version, upcoming_version): - print(f"### Updating schema version {current_schema_version} on {multi_extension_sql_path} " - f"file with the upcoming version {upcoming_version}... ### ") - # TODO Append instead of replace may require - if not append_line_in_file(multi_extension_sql_path, - f"ALTER EXTENSION citus UPDATE TO '{current_schema_version}'", - f"ALTER EXTENSION citus UPDATE TO '{upcoming_minor_version}-1';"): - raise ValueError(f"{multi_extension_sql_path} does not have match for version") - print(f"### Done Current schema version updated on {multi_extension_sql_path} to {upcoming_minor_version}. ###") +def add_downgrade_script_in_multi_extension_file(current_schema_version, + multi_extension_out_path, + upcoming_minor_version, template_file: str): + print(f"### Adding downgrade scripts from version {current_schema_version} to " + f"{upcoming_minor_version} on {multi_extension_out_path}... ### ") + env = get_template_environment(TEMPLATES_PATH) + template = env.get_template( + template_file) # multi_extension_out_prepare_release.tmpl multi_extension_sql_prepare_release.tmpl + string_to_prepend = ( + f"{template.render(current_schema_version=current_schema_version, upcoming_minor_version=f'{upcoming_minor_version}-1')}\n") + + if not prepend_line_in_file(multi_extension_out_path, + f"DROP TABLE prev_objects, extension_diff;", + string_to_prepend): + raise ValueError(f"Downgrade scripts could not be added in {multi_extension_out_path} since " + f"'DROP TABLE prev_objects, extension_diff;' script could not be found ") + print(f"### Done Test downgrade scripts successfully added in {multi_extension_out_path}. ###") def get_current_schema_from_citus_control(citus_control_file_path: str) -> str: @@ -346,6 +380,22 @@ def update_version_in_multi_extension_out(multi_extension_out_path, project_vers print(f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") +def update_detail_strings_in_multi_extension_out(multi_extension_out_path, minor_version): + print(f"### Updating {multi_extension_out_path} detail lines file with the project version {minor_version}... ###") + + if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DETAIL1_PATTERN, + f"{MULTI_EXT_DETAIL_PREFIX}{minor_version}{MULTI_EXT_DETAIL1_SUFFIX}"): + raise ValueError( + f"{multi_extension_out_path} does not contain the version with pattern {MULTI_EXT_DETAIL1_PATTERN}") + + if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DETAIL2_PATTERN, + f"{MULTI_EXT_DETAIL_PREFIX}{minor_version}{MULTI_EXT_DETAIL2_SUFFIX}"): + raise ValueError( + f"{multi_extension_out_path} does not contain the version with pattern {MULTI_EXT_DETAIL2_PATTERN}") + + print(f"### Done {multi_extension_out_path} detail lines updated with project version {minor_version}. ###") + + def update_version_in_multi_extension_out_for_patch(multi_extension_out_path, project_version): print(f"### Updating {multi_extension_out_path} file with the project version {project_version}... ###") @@ -385,10 +435,14 @@ def checkout_branch(branch_name, is_test): print(f"### Done {branch_name} checked out and pulled. ###") +def upgrade_sql_file_name(current_schema_version, upcoming_minor_version): + return f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" + + def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path, upcoming_minor_version): - newly_created_sql_file = f"citus--{current_schema_version}--{upcoming_minor_version}-1.sql" - print(f"### Creating file {newly_created_sql_file}... ###") + newly_created_sql_file = upgrade_sql_file_name(current_schema_version, upcoming_minor_version) + print(f"### Creating upgrade file {newly_created_sql_file}... ###") with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: content = f"/* citus--{current_schema_version}--{upcoming_minor_version}-1 */" content = content + "\n\n" @@ -398,6 +452,22 @@ def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path return newly_created_sql_file +def create_new_sql_for_downgrade_path(current_schema_version, distributed_dir_path, + upcoming_minor_version): + newly_created_sql_file = f"citus--{upcoming_minor_version}--{current_schema_version}-1.sql" + print(f"### Creating downgrade file {newly_created_sql_file}... ###") + with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: + content = f"/* citus--{upcoming_minor_version}--{current_schema_version}-1 */" + content = content + "\n" + content = ( + content + f"-- this is an empty downgrade path since " + f"{upgrade_sql_file_name(current_schema_version, distributed_dir_path)} " + f"is empty for now" + "\n\n") + f_writer.write(content) + print(f"### Done {newly_created_sql_file} created. ###") + return newly_created_sql_file + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--gh_token') @@ -409,6 +479,9 @@ def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path parser.add_argument('--cherry_pick_enabled', nargs='?', default="True") parser.add_argument('--is_test', nargs='?', default="False") arguments = parser.parse_args() + if not arguments.exec_path: + raise ValueError("Exec path should not be empty or null") + os.chdir(arguments.exec_path) update_release(github_token=arguments.gh_token, project_name=arguments.prj_name, project_version=arguments.prj_ver, main_branch=arguments.main_branch, diff --git a/packaging_automation/templates/multi_extension_out_prepare_release.tmpl b/packaging_automation/templates/multi_extension_out_prepare_release.tmpl new file mode 100644 index 00000000..c3dfbf92 --- /dev/null +++ b/packaging_automation/templates/multi_extension_out_prepare_release.tmpl @@ -0,0 +1,17 @@ +-- Test downgrade to {{current_schema_version}} from {{upcoming_minor_version}} +ALTER EXTENSION citus UPDATE TO '{{upcoming_minor_version}}'; +ALTER EXTENSION citus UPDATE TO '{{current_schema_version}}'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at {{upcoming_minor_version}} +ALTER EXTENSION citus UPDATE TO '{{upcoming_minor_version}}'; +SELECT * FROM print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + + diff --git a/packaging_automation/templates/multi_extension_sql_prepare_release.tmpl b/packaging_automation/templates/multi_extension_sql_prepare_release.tmpl new file mode 100644 index 00000000..ac9d3644 --- /dev/null +++ b/packaging_automation/templates/multi_extension_sql_prepare_release.tmpl @@ -0,0 +1,9 @@ +-- Test downgrade to {{current_schema_version}} from {{upcoming_minor_version}} +ALTER EXTENSION citus UPDATE TO '{{upcoming_minor_version}}'; +ALTER EXTENSION citus UPDATE TO '{{current_schema_version}}'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM print_extension_changes(); + +-- Snapshot of state at {{upcoming_minor_version}} +ALTER EXTENSION citus UPDATE TO '{{upcoming_minor_version}}'; +SELECT * FROM print_extension_changes(); diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index bce92426..e2f3c842 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -11,7 +11,7 @@ str_array_to_str, run, remove_text_with_parenthesis, get_version_details, replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, - get_patch_version_regex, append_line_in_file) + get_patch_version_regex, append_line_in_file, prepend_line_in_file) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -88,24 +88,56 @@ def test_get_patch_version_regex(self): self.assertEqual("10\.0\.\d{1,3}", get_patch_version_regex("10.0.3")) def test_append_line_in_file(self): + test_file = "test_append.txt" try: - with open("test_append.txt", "a") as writer: + with open(test_file, "a") as writer: writer.write("Test line 1\n") writer.write("Test line 2\n") - append_line_in_file("test_append.txt", "^Test line 1", "Test line 1.5") - append_line_in_file("test_append.txt", "^Test line 2", "Test line 2.5") - - with open("test_append.txt", "r") as reader: - line1 = reader.readline() - line2 = reader.readline() - line3 = reader.readline() - line4 = reader.readline() - self.assertEqual(line1, "Test line 1\n") - self.assertEqual(line2, "Test line 1.5\n") - self.assertEqual(line3, "Test line 2\n") - self.assertEqual(line4, "Test line 2.5\n") + writer.write("Test line 3\n") + writer.write("Test line 4\n") + writer.write("Test line 5\n") + writer.write("Test line 6\n") + writer.write("Test line 7\n") + writer.write("Test line 8\n") + append_line_in_file(test_file, "^Test line 1", "Test line 1.5") + append_line_in_file(test_file, "^Test line 2", "Test line 2.5") + append_line_in_file(test_file, "^Test line 5", "Test line 5.5") + + with open(test_file, "r") as reader: + lines = reader.readlines() + self.assertEqual(11, len(lines)) + self.assertEqual(lines[0], "Test line 1\n") + self.assertEqual(lines[1], "Test line 1.5\n") + self.assertEqual(lines[2], "Test line 2\n") + self.assertEqual(lines[3], "Test line 2.5\n") finally: - os.remove("test_append.txt") + os.remove(test_file) + + def test_prepend_line_in_file(self): + test_file = "test_prepend.txt" + try: + with open(test_file, "a") as writer: + writer.write("Test line 1\n") + writer.write("Test line 2\n") + writer.write("Test line 3\n") + writer.write("Test line 4\n") + writer.write("Test line 5\n") + writer.write("Test line 6\n") + writer.write("Test line 7\n") + writer.write("Test line 8\n") + prepend_line_in_file(test_file, "^Test line 1", "Test line 0.5") + prepend_line_in_file(test_file, "^Test line 2", "Test line 1.5") + prepend_line_in_file(test_file, "^Test line 5", "Test line 4.5") + + with open(test_file, "r") as reader: + lines = reader.readlines() + self.assertEqual(11, len(lines)) + self.assertEqual(lines[0], "Test line 0.5\n") + self.assertEqual(lines[1], "Test line 1\n") + self.assertEqual(lines[2], "Test line 1.5\n") + self.assertEqual(lines[3], "Test line 2\n") + finally: + os.remove(test_file) if __name__ == '__main__': diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index d6a17c30..db59c1dd 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -24,6 +24,10 @@ def initialize_env(): def test_major_release(): initialize_env() os.chdir(TEST_CHECKOUT_DIR) + + previous_print_extension_changes = count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "SELECT * FROM print_extension_changes();") + try: update_release_return_value = update_release( github_token=github_token, project_name="citus", project_version="10.2.0", main_branch=MAIN_BRANCH, @@ -45,13 +49,20 @@ def test_major_release(): run(f"git checkout {update_release_return_value.upcoming_version_branch}") assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "-- Test downgrade to 10.1-1 from 10.2-1") assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, "ALTER EXTENSION citus UPDATE TO '10.1-1';") + assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "-- Should be empty result since upgrade+downgrade should be a no-op") + assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + "SELECT * FROM print_extension_changes();") - previous_print_extension_changes == 2 assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") + "-- Snapshot of state at 10.2-1") assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_SQL, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") + assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 assert file_includes_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") @@ -64,9 +75,11 @@ def test_major_release(): ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") assert os.path.exists(f"{TEST_BASE_PATH}/{update_release_return_value.upgrade_path_sql_file}") + assert os.path.exists(f"{TEST_BASE_PATH}/{update_release_return_value.downgrade_path_sql_file}") run(f"git checkout {MAIN_BRANCH}") finally: clear_env() + print("test") def test_patch_release(): diff --git a/packaging_automation/update_package_properties.py b/packaging_automation/update_package_properties.py index 83285e64..b7bfeec4 100644 --- a/packaging_automation/update_package_properties.py +++ b/packaging_automation/update_package_properties.py @@ -1,17 +1,16 @@ import argparse import re -from datetime import date, datetime +from dataclasses import dataclass +from datetime import datetime import pathlib2 import string_utils from github import Github -from jinja2 import Environment, FileSystemLoader from parameters_validation import (no_whitespaces, non_blank, non_empty, non_negative, validate_parameters, parameter_validation) -from dataclasses import dataclass from .common_tool_methods import (find_nth_matching_line_and_line_number, find_nth_occurrence_position, - get_project_version_from_tag_name) + get_project_version_from_tag_name, get_template_environment) BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -95,12 +94,6 @@ def spec_file_name(project_name: str) -> str: return f"{project_name}.spec" -def get_template_environment(template_dir: str) -> Environment: - file_loader = FileSystemLoader(template_dir) - env = Environment(loader=file_loader) - return env - - def get_last_changelog_content(all_changelog_content: str) -> str: second_changelog_index = find_nth_occurrence_position(all_changelog_content, "###", 3) changelogs = all_changelog_content[:second_changelog_index] From addc33e55471130f71f698ca7c38717b093e873c Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 2 Jun 2021 12:17:16 +0300 Subject: [PATCH 085/106] * Implement clone in init function * Add schema_version parameter support for patch release tu update citus.control --- packaging_automation/common_tool_methods.py | 5 + packaging_automation/prepare_release.py | 95 +++++++++++++++---- .../tests/test_common_tool_methods.py | 15 ++- .../tests/test_prepare_release.py | 3 +- .../update_package_properties.py | 2 +- 5 files changed, 96 insertions(+), 24 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index e9f57586..ad961c01 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -222,6 +222,11 @@ def get_current_branch() -> str: return repo.active_branch +def does_branch_exist(branch_name: str) -> bool: + repo = Repo(BASE_GIT_PATH) + return branch_name in repo.references + + def get_template_environment(template_dir: str) -> Environment: file_loader = FileSystemLoader(template_dir) env = Environment(loader=file_loader) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 06151b15..ad203b53 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -12,7 +12,8 @@ get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, - append_line_in_file, prepend_line_in_file, get_template_environment) + append_line_in_file, prepend_line_in_file, get_template_environment, + does_branch_exist) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -95,10 +96,12 @@ class PatchReleaseParams: earliest_pr_date: datetime is_test: bool main_branch: str + citus_control_file_path: str multi_extension_out_path: str project_name: str project_version: str release_branch_name: str + schema_version: str repository: Repository @@ -107,8 +110,8 @@ class PatchReleaseParams: def update_release(github_token: non_blank(non_empty(str)), project_name: non_blank(non_empty(str)), project_version: is_version(str), main_branch: non_blank(non_empty(str)), - earliest_pr_date: datetime, exec_path: non_blank(non_empty(str)), is_test: bool = False, - cherry_pick_enabled: bool = False) -> UpdateReleaseReturnValue: + earliest_pr_date: datetime, exec_path: non_blank(non_empty(str)), schema_version: str = "", + is_test: bool = False, cherry_pick_enabled: bool = False) -> UpdateReleaseReturnValue: multi_extension_sql_path = f"{exec_path}/{MULTI_EXTENSION_SQL}" citus_control_file_path = f"{exec_path}/{CITUS_CONTROL}" multi_extension_out_path = f"{exec_path}/{MULTI_EXTENSION_OUT}" @@ -167,6 +170,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl main_branch=main_branch, multi_extension_out_path=multi_extension_out_path, project_name=project_name, project_version=project_version, + schema_version=schema_version, + citus_control_file_path=citus_control_file_path, release_branch_name=release_branch_name, repository=repository) prepare_release_branch_for_patch_release(patch_release_params) return UpdateReleaseReturnValue(release_branch_name, upcoming_version_branch, @@ -176,8 +181,14 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleaseParams): print(f"### {patchReleaseParams.project_version} is a patch release. Executing Patch release flow... ###") - # checkout release branch (release-X.Y) - checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) + # checkout release branch (release-X.Y) In test case release branch for test may not be exist. + # In this case create one + if patchReleaseParams.is_test: + non_test_release_branch = patchReleaseParams.release_branch_name.rstrip("-test") + run(f"git checkout {non_test_release_branch}") + run(f"git checkout -b {patchReleaseParams.release_branch_name}") + else: + checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) # change version info in configure.in file update_version_in_configure_in(patchReleaseParams.configure_in_path, patchReleaseParams.project_version) # execute "auto-conf " @@ -185,6 +196,10 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar # change version info in multi_extension.out update_version_in_multi_extension_out_for_patch(patchReleaseParams.multi_extension_out_path, patchReleaseParams.project_version) + # if schema version is not empty update citus.control schema version + if patchReleaseParams.schema_version: + update_schema_version_in_citus_control(citus_control_file_path=patchReleaseParams.citus_control_file_path, + schema_version=patchReleaseParams.schema_version) if patchReleaseParams.cherry_pick_enabled: # cherry-pick the pr's with backport labels cherrypick_prs_with_backport_labels(patchReleaseParams.earliest_pr_date, patchReleaseParams.main_branch, @@ -238,8 +253,9 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams upcoming_params.upcoming_minor_version) # change version in citus.control file - update_version_with_upcoming_version_in_citus_control(upcoming_params.citus_control_file_path, - upcoming_params.upcoming_minor_version) + default_upcoming_schema_version = f"{upcoming_params.upcoming_minor_version}-1" + update_schema_version_in_citus_control(upcoming_params.citus_control_file_path, + default_upcoming_schema_version) # commit and push changes on master-update-version-$curtime branch commit_changes_for_version_bump(upcoming_params.project_name, upcoming_params.project_version) if not upcoming_params.is_test: @@ -310,12 +326,12 @@ def commit_changes_for_version_bump(project_name, project_version): print(f"### Done Changes committed for {current_branch}. ###") -def update_version_with_upcoming_version_in_citus_control(citus_control_file_path, upcoming_minor_version): - print(f"### Updating {citus_control_file_path} file with the upcoming version {upcoming_minor_version}... ###") +def update_schema_version_in_citus_control(citus_control_file_path, schema_version): + print(f"### Updating {citus_control_file_path} file with the version {schema_version}... ###") if not replace_line_in_file(citus_control_file_path, CITUS_CONTROL_SEARCH_PATTERN, - f"default_version = '{upcoming_minor_version}-1'"): + f"default_version = '{schema_version}'"): raise ValueError(f"{citus_control_file_path} does not have match for version") - print(f"### Done {citus_control_file_path} file is updated with the upcoming version {upcoming_minor_version}. ###") + print(f"### Done {citus_control_file_path} file is updated with the schema version {schema_version}. ###") def add_downgrade_script_in_multi_extension_file(current_schema_version, @@ -432,6 +448,7 @@ def checkout_branch(branch_name, is_test): run(f"git checkout {branch_name}") if not is_test: run(f"git pull") + print(f"### Done {branch_name} checked out and pulled. ###") @@ -468,6 +485,21 @@ def create_new_sql_for_downgrade_path(current_schema_version, distributed_dir_pa return newly_created_sql_file +CHECKOUT_DIR = "citus_temp" + + +def remove_cloned_code(): + if os.path.exists(f"../{CHECKOUT_DIR}"): + os.chdir("..") + run(f"sudo rm -rf {CHECKOUT_DIR}") + + +def initialize_env(): + remove_cloned_code() + if not os.path.exists(CHECKOUT_DIR): + run(f"git clone https://github.com/citusdata/citus.git {CHECKOUT_DIR}") + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--gh_token') @@ -476,15 +508,36 @@ def create_new_sql_for_downgrade_path(current_schema_version, distributed_dir_pa parser.add_argument('--main_branch') parser.add_argument('--earliest_pr_date') parser.add_argument('--exec_path') - parser.add_argument('--cherry_pick_enabled', nargs='?', default="True") + parser.add_argument('--cherry_pick_enabled', nargs='?', default="False") parser.add_argument('--is_test', nargs='?', default="False") + parser.add_argument('--schema_version', nargs='?') arguments = parser.parse_args() - if not arguments.exec_path: - raise ValueError("Exec path should not be empty or null") - os.chdir(arguments.exec_path) - - update_release(github_token=arguments.gh_token, project_name=arguments.prj_name, project_version=arguments.prj_ver, - main_branch=arguments.main_branch, - earliest_pr_date=datetime.strptime(arguments.earliest_pr_date, '%Y.%m.%d %H:%M:%S %z'), - is_test=arguments.is_test.lower() == "true", - cherry_pick_enabled=arguments.cherry_pick_enabled.lower() == "true", exec_path=arguments.exec_path) + is_test = False + try: + initialize_env() + execution_path = f"{os.getcwd()}/{CHECKOUT_DIR}" + major_release = is_major_release(arguments.prj_ver) + + if not arguments.prj_ver and major_release and arguments.cherry_pick_enabled.lower() == "true": + raise ValueError("Cherry-Pick could be enabled only for patch release") + elif not major_release and arguments.cherry_pick_enabled.lower() == "true" \ + and not arguments.earliest_pr_date: + raise ValueError( + "Earliest PR date parameter should not be empty when cherry pick is enabled and release is major.") + earliest_pr_date = None if major_release else datetime.strptime(arguments.earliest_pr_date, + '%Y.%m.%d %H:%M:%S %z') + + os.chdir(execution_path) + + is_test = arguments.is_test.lower() == "true" + + update_release(github_token=arguments.gh_token, project_name=arguments.prj_name, + project_version=arguments.prj_ver, + main_branch=arguments.main_branch, + earliest_pr_date=earliest_pr_date, + is_test=is_test, + cherry_pick_enabled=arguments.cherry_pick_enabled.lower() == "true", exec_path=execution_path, + schema_version=arguments.schema_version) + finally: + if not is_test: + remove_cloned_code() diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index e2f3c842..234936b1 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -11,7 +11,7 @@ str_array_to_str, run, remove_text_with_parenthesis, get_version_details, replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, - get_patch_version_regex, append_line_in_file, prepend_line_in_file) + get_patch_version_regex, append_line_in_file, prepend_line_in_file, does_branch_exist, get_current_branch) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -81,6 +81,19 @@ def test_getprs_with_backlog_label(self): self.assertEqual(1, len(prs_backlog)) self.assertEqual(4746, prs_backlog[0].number) + def test_does_branch_exist(self): + current_branch_name = get_current_branch() + self.assertTrue(does_branch_exist("develop")) + self.assertFalse(does_branch_exist("develop2")) + try: + run("git checkout -b develop-test") + self.assertTrue(does_branch_exist("develop-test")) + run(f"git checkout {current_branch_name} ") + finally: + run(f"git branch -D develop-test ") + + self.assertFalse(does_branch_exist("develop_test")) + def test_get_minor_version(self): self.assertEqual("10.0", get_minor_version("10.0.3")) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index db59c1dd..8bde6c55 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -96,7 +96,7 @@ def test_patch_release(): github_token=github_token, project_name="citus", project_version="10.2.1", main_branch=MAIN_BRANCH, earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True) + exec_path=TEST_BASE_PATH, is_test=True,schema_version="10.2-2") assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") @@ -106,6 +106,7 @@ def test_patch_release(): assert file_includes_line(TEST_BASE_PATH, CONFIGURE, ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") + assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-2'") run(f"git checkout {MAIN_BRANCH}") finally: diff --git a/packaging_automation/update_package_properties.py b/packaging_automation/update_package_properties.py index b7bfeec4..3db607c9 100644 --- a/packaging_automation/update_package_properties.py +++ b/packaging_automation/update_package_properties.py @@ -24,7 +24,7 @@ def is_version(version: str): raise ValueError("version should be non-empty and should not be None") if not re.match(r"\d+\.\d+\.\d+$", version): raise ValueError( - "version should include 3 levels of versions consists of numbers separated with dots. e.g: 10.0.1") + "version should include 3 levels giof versions consists of numbers separated with dots. e.g: 10.0.1") @parameter_validation From 73077181cb73e402e8d05f2468d5000b1ba9c04b Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 2 Jun 2021 13:29:55 +0300 Subject: [PATCH 086/106] * Fix some bugs in corner cases --- packaging_automation/common_tool_methods.py | 4 +-- packaging_automation/prepare_release.py | 34 +++++++++++-------- .../tests/test_common_tool_methods.py | 2 +- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index ad961c01..b194703e 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -217,8 +217,8 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: return has_match -def get_current_branch() -> str: - repo = Repo(BASE_GIT_PATH) +def get_current_branch(working_dir: str) -> str: + repo = Repo(working_dir) return repo.active_branch diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index ad203b53..304354eb 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -319,7 +319,7 @@ def push_branch(upcoming_version_branch): def commit_changes_for_version_bump(project_name, project_version): - current_branch = get_current_branch() + current_branch = get_current_branch(os.getcwd()) print(f"### Committing changes for branch {current_branch}... ###") run("git add .") run(f' git commit -m "Bump {project_name} version to {project_version} "') @@ -438,7 +438,7 @@ def update_version_in_configure_in(configure_in_path, project_version): def create_and_checkout_branch(release_branch_name): - print(f"### Creating release branch with name {release_branch_name} from {get_current_branch()}... ###") + print(f"### Creating release branch with name {release_branch_name} from {get_current_branch(os.getcwd())}... ###") run(f'git checkout -b {release_branch_name}') print(f"### Done {release_branch_name} created. ###") @@ -488,14 +488,16 @@ def create_new_sql_for_downgrade_path(current_schema_version, distributed_dir_pa CHECKOUT_DIR = "citus_temp" -def remove_cloned_code(): - if os.path.exists(f"../{CHECKOUT_DIR}"): +def remove_cloned_code(exec_path: str): + if os.path.exists(f"{exec_path}"): + print("Deleting cloned code ...") os.chdir("..") - run(f"sudo rm -rf {CHECKOUT_DIR}") + run(f"sudo rm -rf {os.path.basename(exec_path)}") + print("Done. Code deleted successfully.") -def initialize_env(): - remove_cloned_code() +def initialize_env(exec_path: str): + remove_cloned_code(exec_path) if not os.path.exists(CHECKOUT_DIR): run(f"git clone https://github.com/citusdata/citus.git {CHECKOUT_DIR}") @@ -513,22 +515,23 @@ def initialize_env(): parser.add_argument('--schema_version', nargs='?') arguments = parser.parse_args() is_test = False + execution_path = f"{os.getcwd()}/{CHECKOUT_DIR}" try: - initialize_env() - execution_path = f"{os.getcwd()}/{CHECKOUT_DIR}" + initialize_env(execution_path) major_release = is_major_release(arguments.prj_ver) - + is_cherry_pick_enabled = arguments.cherry_pick_enabled.lower() == "true" if not arguments.prj_ver and major_release and arguments.cherry_pick_enabled.lower() == "true": raise ValueError("Cherry-Pick could be enabled only for patch release") elif not major_release and arguments.cherry_pick_enabled.lower() == "true" \ and not arguments.earliest_pr_date: raise ValueError( "Earliest PR date parameter should not be empty when cherry pick is enabled and release is major.") - earliest_pr_date = None if major_release else datetime.strptime(arguments.earliest_pr_date, - '%Y.%m.%d %H:%M:%S %z') + earliest_pr_date = None if major_release or not is_cherry_pick_enabled else datetime.strptime( + arguments.earliest_pr_date, + '%Y.%m.%d %H:%M:%S %z') os.chdir(execution_path) - + print(f"Executing in path {execution_path}") is_test = arguments.is_test.lower() == "true" update_release(github_token=arguments.gh_token, project_name=arguments.prj_name, @@ -536,8 +539,9 @@ def initialize_env(): main_branch=arguments.main_branch, earliest_pr_date=earliest_pr_date, is_test=is_test, - cherry_pick_enabled=arguments.cherry_pick_enabled.lower() == "true", exec_path=execution_path, + cherry_pick_enabled=is_cherry_pick_enabled, exec_path=execution_path, schema_version=arguments.schema_version) finally: if not is_test: - remove_cloned_code() + remove_cloned_code(execution_path) + print("waiting") diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 234936b1..3039a470 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -82,7 +82,7 @@ def test_getprs_with_backlog_label(self): self.assertEqual(4746, prs_backlog[0].number) def test_does_branch_exist(self): - current_branch_name = get_current_branch() + current_branch_name = get_current_branch(os.getcwd()) self.assertTrue(does_branch_exist("develop")) self.assertFalse(does_branch_exist("develop2")) try: From 634c93136e187c6e7022b90dc24269f35a733e01 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 2 Jun 2021 13:30:46 +0300 Subject: [PATCH 087/106] * Remove unnecessary logs --- packaging_automation/prepare_release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 304354eb..f12e098c 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -544,4 +544,3 @@ def initialize_env(exec_path: str): finally: if not is_test: remove_cloned_code(execution_path) - print("waiting") From 5e1e9073e6fba070aeb73c7186592eea58beeba6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 3 Jun 2021 11:25:51 +0300 Subject: [PATCH 088/106] Fix round #4 review notes from @onur * Fix cloned directory deletion error * Add repo parameter to support citus-enterprise * Add default value for main branch * Use some argparse features to improve UX * Add some more validations --- packaging_automation/prepare_release.py | 83 ++++++++++++------- .../multi_extension_out_prepare_release.tmpl | 2 - 2 files changed, 53 insertions(+), 32 deletions(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index f12e098c..81090a27 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from datetime import datetime import argparse +from enum import Enum import pathlib2 from github import Github, Repository @@ -12,8 +13,7 @@ get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, - append_line_in_file, prepend_line_in_file, get_template_environment, - does_branch_exist) + prepend_line_in_file, get_template_environment) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -49,6 +49,14 @@ MULTI_EXT_SQL_TEMPLATE_FILE = "multi_extension_sql_prepare_release.tmpl" +class SupportedGithubRepos(Enum): + CITUS = 1 + CITUS_ENTERPRISE = 2 + + +default_project_branches = {"citus": "master", "citus-enterprise": "enterprise-master"} + + @dataclass class UpdateReleaseReturnValue: release_branch_name: str @@ -471,15 +479,15 @@ def create_new_sql_for_upgrade_path(current_schema_version, distributed_dir_path def create_new_sql_for_downgrade_path(current_schema_version, distributed_dir_path, upcoming_minor_version): - newly_created_sql_file = f"citus--{upcoming_minor_version}--{current_schema_version}-1.sql" + newly_created_sql_file = f"citus--{upcoming_minor_version}-1--{current_schema_version}.sql" print(f"### Creating downgrade file {newly_created_sql_file}... ###") with open(f"{distributed_dir_path}/{newly_created_sql_file}", "w") as f_writer: - content = f"/* citus--{upcoming_minor_version}--{current_schema_version}-1 */" + content = f"/* citus--{upcoming_minor_version}-1--{current_schema_version} */" content = content + "\n" content = ( content + f"-- this is an empty downgrade path since " - f"{upgrade_sql_file_name(current_schema_version, distributed_dir_path)} " - f"is empty for now" + "\n\n") + f"{upgrade_sql_file_name(current_schema_version, upcoming_minor_version)} " + f"is empty for now" + "\n") f_writer.write(content) print(f"### Done {newly_created_sql_file} created. ###") return newly_created_sql_file @@ -490,53 +498,68 @@ def create_new_sql_for_downgrade_path(current_schema_version, distributed_dir_pa def remove_cloned_code(exec_path: str): if os.path.exists(f"{exec_path}"): - print("Deleting cloned code ...") - os.chdir("..") - run(f"sudo rm -rf {os.path.basename(exec_path)}") + print(f"Deleting cloned code {exec_path} ...") + # https://stackoverflow.com/questions/51819472/git-cant-delete-local-branch-operation-not-permitted + # https://askubuntu.com/questions/1049142/cannot-delete-git-directory + # since git directory is readonly first we need to give write permission to delete git directory + run(f"chmod -R 777 {exec_path}/.git") + run(f"sudo rm -rf {exec_path}") print("Done. Code deleted successfully.") -def initialize_env(exec_path: str): +def initialize_env(exec_path: str, repo_name: str): remove_cloned_code(exec_path) if not os.path.exists(CHECKOUT_DIR): - run(f"git clone https://github.com/citusdata/citus.git {CHECKOUT_DIR}") + run(f"git clone https://github.com/citusdata/{repo_name}.git {CHECKOUT_DIR}") + + +def validate_parameters(major_release_flag: bool): + if major_release_flag and arguments.cherry_pick_enabled.lower() == "true": + raise ValueError("Cherry pick could be enabled only for patch release") + + if major_release_flag and arguments.earliest_pr_date: + raise ValueError("earliest_pr_date could not be used for major releases") + + if major_release_flag and arguments.schema_version: + raise ValueError("schema_version could not be set for major releases") + + if not major_release_flag and arguments.cherry_pick_enabled.lower() == "true" \ + and not arguments.earliest_pr_date: + raise ValueError( + "earliest_pr_date parameter could not be empty when cherry pick is enabled and release is major.") if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--gh_token') - parser.add_argument('--prj_name') - parser.add_argument('--prj_ver') + parser.add_argument('--gh_token', required=True) + parser.add_argument('--repo', choices=["citus", "citus-enterprise"], required=True) + parser.add_argument('--prj_name', required=True) + parser.add_argument('--prj_ver', required=True) parser.add_argument('--main_branch') parser.add_argument('--earliest_pr_date') - parser.add_argument('--exec_path') parser.add_argument('--cherry_pick_enabled', nargs='?', default="False") - parser.add_argument('--is_test', nargs='?', default="False") + parser.add_argument('--is_test', nargs='?', default="True") parser.add_argument('--schema_version', nargs='?') arguments = parser.parse_args() is_test = False execution_path = f"{os.getcwd()}/{CHECKOUT_DIR}" + major_release = is_major_release(arguments.prj_ver) + validate_parameters(major_release) try: - initialize_env(execution_path) - major_release = is_major_release(arguments.prj_ver) - is_cherry_pick_enabled = arguments.cherry_pick_enabled.lower() == "true" - if not arguments.prj_ver and major_release and arguments.cherry_pick_enabled.lower() == "true": - raise ValueError("Cherry-Pick could be enabled only for patch release") - elif not major_release and arguments.cherry_pick_enabled.lower() == "true" \ - and not arguments.earliest_pr_date: - raise ValueError( - "Earliest PR date parameter should not be empty when cherry pick is enabled and release is major.") - earliest_pr_date = None if major_release or not is_cherry_pick_enabled else datetime.strptime( - arguments.earliest_pr_date, - '%Y.%m.%d %H:%M:%S %z') + initialize_env(execution_path, arguments.repo) + is_cherry_pick_enabled = arguments.cherry_pick_enabled.lower() == "true" + main_branch = arguments.main_branch if arguments.main_branch else default_project_branches[arguments.repo] + print(f"Using main branch {main_branch} for the repo {arguments.repo}.") os.chdir(execution_path) print(f"Executing in path {execution_path}") is_test = arguments.is_test.lower() == "true" - + earliest_pr_date = None if major_release or not is_cherry_pick_enabled else datetime.strptime( + arguments.earliest_pr_date, + '%Y.%m.%d') update_release(github_token=arguments.gh_token, project_name=arguments.prj_name, project_version=arguments.prj_ver, - main_branch=arguments.main_branch, + main_branch=main_branch, earliest_pr_date=earliest_pr_date, is_test=is_test, cherry_pick_enabled=is_cherry_pick_enabled, exec_path=execution_path, diff --git a/packaging_automation/templates/multi_extension_out_prepare_release.tmpl b/packaging_automation/templates/multi_extension_out_prepare_release.tmpl index c3dfbf92..4ca90db5 100644 --- a/packaging_automation/templates/multi_extension_out_prepare_release.tmpl +++ b/packaging_automation/templates/multi_extension_out_prepare_release.tmpl @@ -13,5 +13,3 @@ SELECT * FROM print_extension_changes(); previous_object | current_object --------------------------------------------------------------------- (0 rows) - - From 9c1c198123d4e63c411c27a003e0bff95083ce54 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 3 Jun 2021 11:40:43 +0300 Subject: [PATCH 089/106] Fix unit test error on test release branch creation --- packaging_automation/prepare_release.py | 9 ++++++--- packaging_automation/tests/test_prepare_release.py | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 81090a27..16610e49 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -13,7 +13,7 @@ get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, - prepend_line_in_file, get_template_environment) + does_branch_exist, prepend_line_in_file, get_template_environment) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -193,8 +193,11 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar # In this case create one if patchReleaseParams.is_test: non_test_release_branch = patchReleaseParams.release_branch_name.rstrip("-test") - run(f"git checkout {non_test_release_branch}") - run(f"git checkout -b {patchReleaseParams.release_branch_name}") + if does_branch_exist(non_test_release_branch): + run(f"git checkout {non_test_release_branch}") + run(f"git checkout -b {patchReleaseParams.release_branch_name}") + else: + run(f"git checkout {patchReleaseParams.release_branch_name}") else: checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) # change version info in configure.in file diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 8bde6c55..7aa87fa4 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -83,7 +83,6 @@ def test_major_release(): def test_patch_release(): - print(os.getcwd()) initialize_env() os.chdir(TEST_CHECKOUT_DIR) try: From ae378096b1bee539730ebbc89f020796d14d2f95 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 3 Jun 2021 13:34:43 +0300 Subject: [PATCH 090/106] Use store_true argument for boolean values --- packaging_automation/prepare_release.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 16610e49..c39b6d75 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -540,14 +540,16 @@ def validate_parameters(major_release_flag: bool): parser.add_argument('--prj_ver', required=True) parser.add_argument('--main_branch') parser.add_argument('--earliest_pr_date') - parser.add_argument('--cherry_pick_enabled', nargs='?', default="False") - parser.add_argument('--is_test', nargs='?', default="True") + parser.add_argument('--cherry_pick_enabled', action="store_true") + parser.add_argument('--is_test', action="store_true") parser.add_argument('--schema_version', nargs='?') arguments = parser.parse_args() is_test = False execution_path = f"{os.getcwd()}/{CHECKOUT_DIR}" major_release = is_major_release(arguments.prj_ver) + print("Cherry pick enabled: " + str(arguments.cherry_pick_enabled)) validate_parameters(major_release) + try: initialize_env(execution_path, arguments.repo) From 080f34a93feeff63e59718088fe6105e555dbd91 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 3 Jun 2021 13:46:40 +0300 Subject: [PATCH 091/106] Remove unnecessary log --- packaging_automation/prepare_release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index c39b6d75..59954165 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -547,7 +547,6 @@ def validate_parameters(major_release_flag: bool): is_test = False execution_path = f"{os.getcwd()}/{CHECKOUT_DIR}" major_release = is_major_release(arguments.prj_ver) - print("Cherry pick enabled: " + str(arguments.cherry_pick_enabled)) validate_parameters(major_release) try: From 616f253abbeb84d235c7dbb206b1d45a948c02dd Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 4 Jun 2021 05:19:07 +0300 Subject: [PATCH 092/106] Add remote and local branch checks --- packaging_automation/common_tool_methods.py | 21 +++++++-- packaging_automation/prepare_release.py | 44 +++++++++++-------- .../tests/test_common_tool_methods.py | 29 +++++++++--- .../tests/test_prepare_release.py | 4 +- 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b194703e..2cc3595a 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -222,9 +222,24 @@ def get_current_branch(working_dir: str) -> str: return repo.active_branch -def does_branch_exist(branch_name: str) -> bool: - repo = Repo(BASE_GIT_PATH) - return branch_name in repo.references +def does_remote_branch_exist(branch_name: str, working_dir: str) -> bool: + repo = Repo(working_dir) + for rp in repo.references: + if rp.name.endswith(f"/{branch_name}"): + return True + return False + + +def does_local_branch_exist(branch_name: str, working_dir: str) -> bool: + repo = Repo(working_dir) + for rp in repo.branches: + if rp.name == branch_name: + return True + return False + + +def does_branch_exist(branch_name: str, working_dir: str) -> bool: + return does_local_branch_exist(branch_name, working_dir) or does_remote_branch_exist(branch_name, working_dir) def get_template_environment(template_dir: str) -> Environment: diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 59954165..c7a3fec7 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -13,7 +13,7 @@ get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, - does_branch_exist, prepend_line_in_file, get_template_environment) + does_remote_branch_exist, prepend_line_in_file, get_template_environment) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -192,12 +192,21 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar # checkout release branch (release-X.Y) In test case release branch for test may not be exist. # In this case create one if patchReleaseParams.is_test: + non_test_release_branch = patchReleaseParams.release_branch_name.rstrip("-test") - if does_branch_exist(non_test_release_branch): + release_branch_exist = does_remote_branch_exist(non_test_release_branch, os.getcwd()) + test_release_branch_exist = does_remote_branch_exist(patchReleaseParams.release_branch_name, os.getcwd()) + print("Non test release branch " + non_test_release_branch) + print("Non test release branch exist" + str(release_branch_exist)) + print("Non test release branch exist1" + str(release_branch_exist)) + print ("where am i: ",os.getcwd()) + if does_remote_branch_exist(non_test_release_branch, os.getcwd()): run(f"git checkout {non_test_release_branch}") run(f"git checkout -b {patchReleaseParams.release_branch_name}") - else: + elif test_release_branch_exist: run(f"git checkout {patchReleaseParams.release_branch_name}") + else: + run(f"git checkout -b {patchReleaseParams.release_branch_name}") else: checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) # change version info in configure.in file @@ -403,7 +412,7 @@ def update_version_in_multi_extension_out(multi_extension_out_path, project_vers if not replace_line_in_file(multi_extension_out_path, MULTI_EXT_DEVEL_SEARCH_PATTERN, f" {project_version}"): raise ValueError( - f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") + f"{multi_extension_out_path} does not contain the version with pattern {MULTI_EXT_DEVEL_SEARCH_PATTERN}") print(f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") @@ -430,7 +439,7 @@ def update_version_in_multi_extension_out_for_patch(multi_extension_out_path, pr get_patch_version_regex(project_version), f" {project_version}"): raise ValueError( - f"{multi_extension_out_path} does not contain the version with pattern {CONFIGURE_IN_SEARCH_PATTERN}") + f"{multi_extension_out_path} does not contain the version with pattern {get_patch_version_regex(project_version)}") print(f"### Done {multi_extension_out_path} file is updated with project version {project_version}. ###") @@ -510,14 +519,14 @@ def remove_cloned_code(exec_path: str): print("Done. Code deleted successfully.") -def initialize_env(exec_path: str, repo_name: str): +def initialize_env(exec_path: str, project_name: str): remove_cloned_code(exec_path) if not os.path.exists(CHECKOUT_DIR): - run(f"git clone https://github.com/citusdata/{repo_name}.git {CHECKOUT_DIR}") + run(f"git clone https://github.com/citusdata/{project_name}.git {CHECKOUT_DIR}") def validate_parameters(major_release_flag: bool): - if major_release_flag and arguments.cherry_pick_enabled.lower() == "true": + if major_release_flag and arguments.cherry_pick_enabled: raise ValueError("Cherry pick could be enabled only for patch release") if major_release_flag and arguments.earliest_pr_date: @@ -526,7 +535,7 @@ def validate_parameters(major_release_flag: bool): if major_release_flag and arguments.schema_version: raise ValueError("schema_version could not be set for major releases") - if not major_release_flag and arguments.cherry_pick_enabled.lower() == "true" \ + if not major_release_flag and arguments.cherry_pick_enabled \ and not arguments.earliest_pr_date: raise ValueError( "earliest_pr_date parameter could not be empty when cherry pick is enabled and release is major.") @@ -535,8 +544,7 @@ def validate_parameters(major_release_flag: bool): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--gh_token', required=True) - parser.add_argument('--repo', choices=["citus", "citus-enterprise"], required=True) - parser.add_argument('--prj_name', required=True) + parser.add_argument('--prj_name', choices=["citus", "citus-enterprise"], required=True) parser.add_argument('--prj_ver', required=True) parser.add_argument('--main_branch') parser.add_argument('--earliest_pr_date') @@ -550,14 +558,14 @@ def validate_parameters(major_release_flag: bool): validate_parameters(major_release) try: - initialize_env(execution_path, arguments.repo) + initialize_env(execution_path, arguments.prj_name) - is_cherry_pick_enabled = arguments.cherry_pick_enabled.lower() == "true" - main_branch = arguments.main_branch if arguments.main_branch else default_project_branches[arguments.repo] - print(f"Using main branch {main_branch} for the repo {arguments.repo}.") + is_cherry_pick_enabled = arguments.cherry_pick_enabled + main_branch = arguments.main_branch if arguments.main_branch else default_project_branches[arguments.prj_name] + print(f"Using main branch {main_branch} for the repo {arguments.prj_name}.") os.chdir(execution_path) print(f"Executing in path {execution_path}") - is_test = arguments.is_test.lower() == "true" + is_test = arguments.is_test earliest_pr_date = None if major_release or not is_cherry_pick_enabled else datetime.strptime( arguments.earliest_pr_date, '%Y.%m.%d') @@ -565,8 +573,8 @@ def validate_parameters(major_release_flag: bool): project_version=arguments.prj_ver, main_branch=main_branch, earliest_pr_date=earliest_pr_date, - is_test=is_test, - cherry_pick_enabled=is_cherry_pick_enabled, exec_path=execution_path, + is_test=arguments.is_test, + cherry_pick_enabled=arguments.cherry_pick_enabled, exec_path=execution_path, schema_version=arguments.schema_version) finally: if not is_test: diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 3039a470..3166dc25 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -1,17 +1,20 @@ import os import unittest +import uuid from shutil import copyfile import pathlib2 from github import Github from datetime import datetime +import uuid from ..common_tool_methods import ( find_nth_occurrence_position, is_major_release, str_array_to_str, run, remove_text_with_parenthesis, get_version_details, replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, - get_patch_version_regex, append_line_in_file, prepend_line_in_file, does_branch_exist, get_current_branch) + get_patch_version_regex, append_line_in_file, prepend_line_in_file, does_remote_branch_exist, get_current_branch, + does_local_branch_exist) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -81,18 +84,32 @@ def test_getprs_with_backlog_label(self): self.assertEqual(1, len(prs_backlog)) self.assertEqual(4746, prs_backlog[0].number) - def test_does_branch_exist(self): + def test_does_local_branch_exist(self): current_branch_name = get_current_branch(os.getcwd()) - self.assertTrue(does_branch_exist("develop")) - self.assertFalse(does_branch_exist("develop2")) + self.assertTrue(does_remote_branch_exist("develop", os.getcwd())) + self.assertFalse(does_remote_branch_exist("develop2", os.getcwd())) try: run("git checkout -b develop-test") - self.assertTrue(does_branch_exist("develop-test")) + self.assertTrue(does_local_branch_exist("develop-test", os.getcwd())) run(f"git checkout {current_branch_name} ") finally: run(f"git branch -D develop-test ") - self.assertFalse(does_branch_exist("develop_test")) + self.assertFalse(does_remote_branch_exist("develop_test", os.getcwd())) + + def test_does_remote_branch_exist(self): + current_branch_name = get_current_branch(os.getcwd()) + branch_name = "develop-remote-test" + try: + run(f"git checkout develop") + run(f"git checkout -b {branch_name}") + run(f"git push --set-upstream origin {branch_name}") + self.assertTrue(does_remote_branch_exist(branch_name, os.getcwd())) + self.assertFalse(does_remote_branch_exist(f"{branch_name}{uuid.uuid4()}", os.getcwd())) + finally: + run(f"git checkout {current_branch_name} ") + run(f"git branch -D {branch_name}") + run(f"git push origin --delete {branch_name}") def test_get_minor_version(self): self.assertEqual("10.0", get_minor_version("10.0.3")) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 7aa87fa4..f807af73 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -10,7 +10,7 @@ BASE_PATH = pathlib2.Path(__file__).parents[2] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") -MAIN_BRANCH = "test-tools-scripts" +MAIN_BRANCH = "enterprise-master" TEST_CHECKOUT_DIR = "citus_test" TEST_BASE_PATH = f"{BASE_PATH}/{TEST_CHECKOUT_DIR}" @@ -18,7 +18,7 @@ def initialize_env(): clear_env() if not os.path.exists(TEST_CHECKOUT_DIR): - run(f"git clone https://github.com/citusdata/citus.git {TEST_CHECKOUT_DIR}") + run(f"git clone https://github.com/citusdata/citus-enterprise.git {TEST_CHECKOUT_DIR}") def test_major_release(): From 48768d93b5e065255da42f1d4ec4ac9d2bda85a2 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 4 Jun 2021 05:30:21 +0300 Subject: [PATCH 093/106] Fix unit test for local branch check --- packaging_automation/tests/test_common_tool_methods.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 3166dc25..e62072b3 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -86,14 +86,15 @@ def test_getprs_with_backlog_label(self): def test_does_local_branch_exist(self): current_branch_name = get_current_branch(os.getcwd()) + branch_name = "develop-local-test" self.assertTrue(does_remote_branch_exist("develop", os.getcwd())) self.assertFalse(does_remote_branch_exist("develop2", os.getcwd())) try: - run("git checkout -b develop-test") - self.assertTrue(does_local_branch_exist("develop-test", os.getcwd())) + run(f"git checkout -b {branch_name}") + self.assertTrue(does_local_branch_exist(branch_name, os.getcwd())) run(f"git checkout {current_branch_name} ") finally: - run(f"git branch -D develop-test ") + run(f"git branch -D {branch_name}") self.assertFalse(does_remote_branch_exist("develop_test", os.getcwd())) From ee70e54826d482c5f5a7b228fbd4b27ed3e24a15 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 4 Jun 2021 05:32:23 +0300 Subject: [PATCH 094/106] Fix unit test for remote branch check --- packaging_automation/tests/test_common_tool_methods.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index e62072b3..d52f98e7 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -102,14 +102,20 @@ def test_does_remote_branch_exist(self): current_branch_name = get_current_branch(os.getcwd()) branch_name = "develop-remote-test" try: + try: + run(f"git branch -D {branch_name}") + except: + print(f"{branch_name} already deleted ") run(f"git checkout develop") run(f"git checkout -b {branch_name}") run(f"git push --set-upstream origin {branch_name}") + run(f"git checkout develop") + run(f"git branch -D {branch_name}") self.assertTrue(does_remote_branch_exist(branch_name, os.getcwd())) self.assertFalse(does_remote_branch_exist(f"{branch_name}{uuid.uuid4()}", os.getcwd())) finally: run(f"git checkout {current_branch_name} ") - run(f"git branch -D {branch_name}") + # run(f"git branch -D {branch_name}") run(f"git push origin --delete {branch_name}") def test_get_minor_version(self): From 3ba99efb54a7bdbbade627369af2c6e788d6dde8 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 4 Jun 2021 05:43:51 +0300 Subject: [PATCH 095/106] Add remote and local branch checks on prepare release --- packaging_automation/prepare_release.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index c7a3fec7..26e7858e 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -13,7 +13,8 @@ get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, - does_remote_branch_exist, prepend_line_in_file, get_template_environment) + does_remote_branch_exist, does_local_branch_exist, prepend_line_in_file, + get_template_environment) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -195,12 +196,9 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar non_test_release_branch = patchReleaseParams.release_branch_name.rstrip("-test") release_branch_exist = does_remote_branch_exist(non_test_release_branch, os.getcwd()) - test_release_branch_exist = does_remote_branch_exist(patchReleaseParams.release_branch_name, os.getcwd()) - print("Non test release branch " + non_test_release_branch) - print("Non test release branch exist" + str(release_branch_exist)) - print("Non test release branch exist1" + str(release_branch_exist)) - print ("where am i: ",os.getcwd()) - if does_remote_branch_exist(non_test_release_branch, os.getcwd()): + test_release_branch_exist = does_local_branch_exist(patchReleaseParams.release_branch_name, os.getcwd()) + + if release_branch_exist: run(f"git checkout {non_test_release_branch}") run(f"git checkout -b {patchReleaseParams.release_branch_name}") elif test_release_branch_exist: From bfffcb8719bbc198e2222fb29f792dfc8f351487 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 7 Jun 2021 07:02:36 +0300 Subject: [PATCH 096/106] Use citus as test source --- packaging_automation/tests/test_prepare_release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index f807af73..bca2e206 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -10,7 +10,7 @@ BASE_PATH = pathlib2.Path(__file__).parents[2] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") -MAIN_BRANCH = "enterprise-master" +MAIN_BRANCH = "master" TEST_CHECKOUT_DIR = "citus_test" TEST_BASE_PATH = f"{BASE_PATH}/{TEST_CHECKOUT_DIR}" @@ -18,7 +18,7 @@ def initialize_env(): clear_env() if not os.path.exists(TEST_CHECKOUT_DIR): - run(f"git clone https://github.com/citusdata/citus-enterprise.git {TEST_CHECKOUT_DIR}") + run(f"git clone https://github.com/citusdata/citus.git {TEST_CHECKOUT_DIR}") def test_major_release(): From 3aac3298d0017ea9c41d84213b5d0e861bf8bbf0 Mon Sep 17 00:00:00 2001 From: Gurkan Indibay Date: Wed, 9 Jun 2021 03:12:42 +0000 Subject: [PATCH 097/106] Fix version update error on major execution --- citus_temp | 1 + packaging_automation/common_tool_methods.py | 10 +++++-- packaging_automation/prepare_release.py | 28 +++++++++---------- packaging_automation/tests/test.py | 5 ++++ .../tests/test_common_tool_methods.py | 25 +++++++++++++++-- .../tests/test_prepare_release.py | 21 +++++++------- test1e58b070-b0bb-4f78-9ae4-3672bd131097 | 1 + test413c9add-f28a-417c-8fbc-ff20264626f5 | 1 + testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 | 1 + 9 files changed, 64 insertions(+), 29 deletions(-) create mode 160000 citus_temp create mode 100644 packaging_automation/tests/test.py create mode 100644 test1e58b070-b0bb-4f78-9ae4-3672bd131097 create mode 100644 test413c9add-f28a-417c-8fbc-ff20264626f5 create mode 100644 testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 diff --git a/citus_temp b/citus_temp new file mode 160000 index 00000000..09193124 --- /dev/null +++ b/citus_temp @@ -0,0 +1 @@ +Subproject commit 091931247402ee661d48c483b40f8be485df7b6d diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 2cc3595a..ea5f9cf4 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -96,8 +96,14 @@ def get_upcoming_patch_version(version: is_version(str)) -> str: def get_upcoming_minor_version(version: is_version(str)) -> str: - upcoming_version_details = get_version_details(version) - return f'{upcoming_version_details["major"]}.{upcoming_version_details["minor"]}' + project_version_details = get_version_details(version) + return f'{project_version_details["major"]}.{int(project_version_details["minor"]) + 1}' + + +def get_last_commit_message(path: str) -> str: + repo = Repo("/vagrant/release/tools") + commit = repo.head.commit + return commit.message def is_major_release(version: is_version(str)) -> bool: diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 26e7858e..6f7c7868 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -14,7 +14,7 @@ filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, does_remote_branch_exist, does_local_branch_exist, prepend_line_in_file, - get_template_environment) + get_template_environment, get_upcoming_minor_version) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -83,7 +83,7 @@ class UpcomingVersionBranchParams: citus_control_file_path: str config_py_path: str configure_in_path: str - devel_version: str + upcoming_devel_version: str distributed_dir_path: str downgrades_dir_path: str is_test: bool @@ -94,7 +94,6 @@ class UpcomingVersionBranchParams: project_version: str repository: Repository upcoming_minor_version: str - upcoming_version: str upcoming_version_branch: str @@ -130,10 +129,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl downgrades_dir_path = f"{exec_path}/{DOWNGRADES_DIR_PATH}" project_version_details = get_version_details(project_version) - default_upcoming_version = get_upcoming_patch_version(project_version) - upcoming_version = os.getenv("UPCOMING_VERSION", default=default_upcoming_version) - upcoming_minor_version = get_minor_version(upcoming_version) - devel_version = f"{upcoming_minor_version}devel" + upcoming_minor_version = get_upcoming_minor_version(project_version) + upcoming_devel_version = f"{upcoming_minor_version}devel" release_branch_name = f'release-{project_version_details["major"]}.{project_version_details["minor"]}' release_branch_name = f"{release_branch_name}-test" if is_test else release_branch_name @@ -147,7 +144,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl # major release if is_major_release(project_version): print(f"### {project_version} is a major release. Executing Major release flow... ###") - major_release_params = MajorReleaseParams(configure_in_path=configure_in_path, devel_version=devel_version, + major_release_params = MajorReleaseParams(configure_in_path=configure_in_path, + devel_version=upcoming_devel_version, is_test=is_test, main_branch=main_branch, multi_extension_out_path=multi_extension_out_path, project_name=project_name, project_version=project_version, @@ -155,9 +153,8 @@ def update_release(github_token: non_blank(non_empty(str)), project_name: non_bl prepare_release_branch_for_major_release(major_release_params) branch_params = UpcomingVersionBranchParams(project_version=project_version, project_name=project_name, - upcoming_version=upcoming_version, upcoming_version_branch=upcoming_version_branch, - devel_version=devel_version, is_test=is_test, + upcoming_devel_version=upcoming_devel_version, is_test=is_test, main_branch=main_branch, citus_control_file_path=citus_control_file_path, config_py_path=config_py_path, @@ -240,14 +237,15 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams # create master-update-version-$curtime branch create_and_checkout_branch(upcoming_params.upcoming_version_branch) # update version info with upcoming version on configure.in - update_version_in_configure_in(upcoming_params.configure_in_path, upcoming_params.devel_version) + update_version_in_configure_in(upcoming_params.configure_in_path, upcoming_params.upcoming_devel_version) # update version info with upcoming version on config.py update_version_with_upcoming_version_in_config_py(upcoming_params.config_py_path, upcoming_params.upcoming_minor_version) # execute autoconf -f execute_autoconf_f() - # update version info with upcoming version on multiextension.out - update_version_in_multi_extension_out(upcoming_params.multi_extension_out_path, upcoming_params.devel_version) + # update version info with upcoming version on multi_extension.out + update_version_in_multi_extension_out(upcoming_params.multi_extension_out_path, + upcoming_params.upcoming_devel_version) # update detail lines with minor version update_detail_strings_in_multi_extension_out(upcoming_params.multi_extension_out_path, upcoming_params.upcoming_minor_version) @@ -275,14 +273,14 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams update_schema_version_in_citus_control(upcoming_params.citus_control_file_path, default_upcoming_schema_version) # commit and push changes on master-update-version-$curtime branch - commit_changes_for_version_bump(upcoming_params.project_name, upcoming_params.project_version) + commit_changes_for_version_bump(upcoming_params.project_name, upcoming_params.upcoming_devel_version) if not upcoming_params.is_test: push_branch(upcoming_params.upcoming_version_branch) # create pull request create_pull_request_for_upcoming_version_branch(upcoming_params.repository, upcoming_params.main_branch, upcoming_params.upcoming_version_branch, - upcoming_params.upcoming_version) + upcoming_params.upcoming_devel_version) print(f"### Done {upcoming_params.upcoming_version_branch} flow executed. ###") return upgrade_file, downgrade_file diff --git a/packaging_automation/tests/test.py b/packaging_automation/tests/test.py new file mode 100644 index 00000000..94fff2fe --- /dev/null +++ b/packaging_automation/tests/test.py @@ -0,0 +1,5 @@ + +import git +repo = git.Repo("/vagrant/release/tools") +commit = repo.head.commit +print(commit.message) \ No newline at end of file diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index d52f98e7..21347c82 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -11,10 +11,10 @@ from ..common_tool_methods import ( find_nth_occurrence_position, is_major_release, str_array_to_str, run, remove_text_with_parenthesis, get_version_details, - replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, + replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label,get_upcoming_minor_version, get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, append_line_in_file, prepend_line_in_file, does_remote_branch_exist, get_current_branch, - does_local_branch_exist) + does_local_branch_exist,get_last_commit_message) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -46,6 +46,8 @@ def test_run(self): result = run("echo 'Run' method is performing fine ") self.assertEqual(0, result.returncode) + + def test_remove_paranthesis_from_string(self): self.assertEqual("out of paranthesis ", remove_text_with_parenthesis("out of paranthesis (inside paranthesis)")) @@ -66,6 +68,25 @@ def test_replace_line_in_file(self): finally: os.remove(copy_file_path) + def test_get_upcoming_minor_version(self): + assert get_upcoming_minor_version("10.1.0") == "10.2" + + def test_get_last_commit_message(self): + current_branch_name = get_current_branch(os.getcwd()) + test_branch_name = f"test{uuid.uuid4()}" + run(f"git checkout -b {test_branch_name}") + try: + with open(test_branch_name,"w") as writer: + writer.write("Test content") + run(f"git add .") + commit_message = f"Test message for {test_branch_name}" + run(f"git commit -m '{commit_message}'") + assert get_last_commit_message(os.getcwd()) == f"{commit_message}\n" + finally: + run(f"git checkout {current_branch_name}") + run(f"git branch -D {test_branch_name}") + + def test_getprs(self): # created at is not seen on Github. Should be checked on API result g = Github(GITHUB_TOKEN) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index bca2e206..759ec5ef 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -4,7 +4,7 @@ from ..prepare_release import (update_release, MULTI_EXTENSION_OUT, MULTI_EXTENSION_SQL, CONFIGURE, CONFIGURE_IN, CITUS_CONTROL, CONFIG_PY) -from ..common_tool_methods import file_includes_line, count_line_in_file, run +from ..common_tool_methods import file_includes_line, count_line_in_file, run,get_last_commit_message github_token = os.getenv("GH_TOKEN") @@ -30,25 +30,25 @@ def test_major_release(): try: update_release_return_value = update_release( - github_token=github_token, project_name="citus", project_version="10.2.0", main_branch=MAIN_BRANCH, + github_token=github_token, project_name="citus", project_version="10.1.0", main_branch=MAIN_BRANCH, earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), exec_path=TEST_BASE_PATH, is_test=True) run(f"git checkout {update_release_return_value.release_branch_name}") - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.0") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.0])") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.0'") + assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.1.0") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.1.0])") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.1.0'") + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.1.0'") assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2.0 to adapt to many kinds of systems.") + r"\`configure' configures Citus 10.1.0 to adapt to many kinds of systems.") assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.0:";;') - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.0'") + ' short | recursive ) echo "Configuration of Citus 10.1.0:";;') + assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.1.0'") run(f"git checkout {update_release_return_value.upcoming_version_branch}") - assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-1'") + assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.1-1'") assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, "-- Test downgrade to 10.1-1 from 10.2-1") assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, @@ -76,6 +76,7 @@ def test_major_release(): assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") assert os.path.exists(f"{TEST_BASE_PATH}/{update_release_return_value.upgrade_path_sql_file}") assert os.path.exists(f"{TEST_BASE_PATH}/{update_release_return_value.downgrade_path_sql_file}") + assert get_last_commit_message(os.getcwd()) == "Bump Citus to 10.2devel\n" run(f"git checkout {MAIN_BRANCH}") finally: clear_env() diff --git a/test1e58b070-b0bb-4f78-9ae4-3672bd131097 b/test1e58b070-b0bb-4f78-9ae4-3672bd131097 new file mode 100644 index 00000000..3f3f005b --- /dev/null +++ b/test1e58b070-b0bb-4f78-9ae4-3672bd131097 @@ -0,0 +1 @@ +Test content \ No newline at end of file diff --git a/test413c9add-f28a-417c-8fbc-ff20264626f5 b/test413c9add-f28a-417c-8fbc-ff20264626f5 new file mode 100644 index 00000000..3f3f005b --- /dev/null +++ b/test413c9add-f28a-417c-8fbc-ff20264626f5 @@ -0,0 +1 @@ +Test content \ No newline at end of file diff --git a/testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 b/testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 new file mode 100644 index 00000000..3f3f005b --- /dev/null +++ b/testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 @@ -0,0 +1 @@ +Test content \ No newline at end of file From 81a4413c689070f1f77cb94d5e9b5940efeb013f Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 9 Jun 2021 12:44:40 +0300 Subject: [PATCH 098/106] Fix unit test errors while teardown test resources --- citus_temp | 1 - packaging_automation/common_tool_methods.py | 46 ++++- packaging_automation/prepare_release.py | 3 +- .../tests/test_prepare_release.py | 180 +++++++++--------- 4 files changed, 129 insertions(+), 101 deletions(-) delete mode 160000 citus_temp diff --git a/citus_temp b/citus_temp deleted file mode 160000 index 09193124..00000000 --- a/citus_temp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 091931247402ee661d48c483b40f8be485df7b6d diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index ea5f9cf4..841a2384 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -1,20 +1,33 @@ +import os import re import subprocess from datetime import datetime from typing import Dict, List -import os +from typing import Tuple +import pathlib2 +from git import Repo from github import Repository, PullRequest, Commit from jinja2 import Environment, FileSystemLoader from .common_validations import (is_tag, is_version) -from git import Repo -import pathlib2 -from typing import Tuple BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX = "\.\d{1,3}" +referenced_repos:List[Repo] = [] + + +def get_new_repo(working_dir: str) -> Repo: + repo = Repo(working_dir) + referenced_repos.append(repo) + return repo + + +def release_all_repos(): + for repo in referenced_repos: + repo.close() + def get_spec_file_name(project_name: str) -> str: return f"{project_name}.spec" @@ -101,7 +114,7 @@ def get_upcoming_minor_version(version: is_version(str)) -> str: def get_last_commit_message(path: str) -> str: - repo = Repo("/vagrant/release/tools") + repo = get_new_repo(path) commit = repo.head.commit return commit.message @@ -224,12 +237,12 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: def get_current_branch(working_dir: str) -> str: - repo = Repo(working_dir) + repo = get_new_repo(working_dir) return repo.active_branch def does_remote_branch_exist(branch_name: str, working_dir: str) -> bool: - repo = Repo(working_dir) + repo = get_new_repo(working_dir) for rp in repo.references: if rp.name.endswith(f"/{branch_name}"): return True @@ -237,7 +250,7 @@ def does_remote_branch_exist(branch_name: str, working_dir: str) -> bool: def does_local_branch_exist(branch_name: str, working_dir: str) -> bool: - repo = Repo(working_dir) + repo = get_new_repo(working_dir) for rp in repo.branches: if rp.name == branch_name: return True @@ -252,3 +265,20 @@ def get_template_environment(template_dir: str) -> Environment: file_loader = FileSystemLoader(template_dir) env = Environment(loader=file_loader) return env + + +def remove_cloned_code(exec_path: str): + release_all_repos() + if os.path.exists(f"{exec_path}"): + print(f"Deleting cloned code {exec_path} ...") + # https://stackoverflow.com/questions/51819472/git-cant-delete-local-branch-operation-not-permitted + # https://askubuntu.com/questions/1049142/cannot-delete-git-directory + # since git directory is readonly first we need to give write permission to delete git directory + if os.path.exists(f"{exec_path}/.git"): + run(f"chmod -R 777 {exec_path}/.git") + try: + run(f"sudo rm -rf {exec_path}") + print("Done. Code deleted successfully.") + except: + print(f"Some files could not be deleted in directory {exec_path}. " + f"Please delete them manually or they will be deleted before next execution") diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 6f7c7868..2092da90 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -14,7 +14,7 @@ filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, does_remote_branch_exist, does_local_branch_exist, prepend_line_in_file, - get_template_environment, get_upcoming_minor_version) + get_template_environment, get_upcoming_minor_version,remove_cloned_code) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -575,3 +575,4 @@ def validate_parameters(major_release_flag: bool): finally: if not is_test: remove_cloned_code(execution_path) + diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 759ec5ef..335688b0 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -1,119 +1,117 @@ -import pathlib2 import os +import uuid from datetime import datetime +import time + +import pathlib2 +import shutil +from ..common_tool_methods import (file_includes_line, count_line_in_file, run, get_last_commit_message, + remove_cloned_code) from ..prepare_release import (update_release, MULTI_EXTENSION_OUT, MULTI_EXTENSION_SQL, CONFIGURE, CONFIGURE_IN, CITUS_CONTROL, CONFIG_PY) -from ..common_tool_methods import file_includes_line, count_line_in_file, run,get_last_commit_message github_token = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[2] if os.getenv("BASE_PATH") is None else os.getenv("BASE_PATH") -MAIN_BRANCH = "master" +MAIN_BRANCH = "test-tools-scripts" TEST_CHECKOUT_DIR = "citus_test" -TEST_BASE_PATH = f"{BASE_PATH}/{TEST_CHECKOUT_DIR}" +resources_to_be_deleted = [] -def initialize_env(): - clear_env() - if not os.path.exists(TEST_CHECKOUT_DIR): - run(f"git clone https://github.com/citusdata/citus.git {TEST_CHECKOUT_DIR}") + +def initialize_env() -> str: + test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" + remove_cloned_code(test_base_path_major) + if not os.path.exists(test_base_path_major): + run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") + return test_base_path_major def test_major_release(): - initialize_env() - os.chdir(TEST_CHECKOUT_DIR) + test_base_path_major = initialize_env() + os.chdir(test_base_path_major) + resources_to_be_deleted.append(test_base_path_major) - previous_print_extension_changes = count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_OUT, + previous_print_extension_changes = count_line_in_file(test_base_path_major, MULTI_EXTENSION_OUT, "SELECT * FROM print_extension_changes();") - try: - update_release_return_value = update_release( - github_token=github_token, project_name="citus", project_version="10.1.0", main_branch=MAIN_BRANCH, - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True) - - run(f"git checkout {update_release_return_value.release_branch_name}") - - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.1.0") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.1.0])") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.1.0'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.1.0'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.1.0 to adapt to many kinds of systems.") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.1.0:";;') - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.1.0'") - - run(f"git checkout {update_release_return_value.upcoming_version_branch}") - - assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.1-1'") - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "-- Test downgrade to 10.1-1 from 10.2-1") - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "ALTER EXTENSION citus UPDATE TO '10.1-1';") - assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "-- Should be empty result since upgrade+downgrade should be a no-op") - assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "SELECT * FROM print_extension_changes();") - previous_print_extension_changes == 2 - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, - "-- Snapshot of state at 10.2-1") - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2devel") - - assert count_line_in_file(TEST_BASE_PATH, MULTI_EXTENSION_SQL, - "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 - assert file_includes_line(TEST_BASE_PATH, CONFIG_PY, "MASTER_VERSION = '10.2'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2devel'") - assert os.path.exists(f"{TEST_BASE_PATH}/{update_release_return_value.upgrade_path_sql_file}") - assert os.path.exists(f"{TEST_BASE_PATH}/{update_release_return_value.downgrade_path_sql_file}") - assert get_last_commit_message(os.getcwd()) == "Bump Citus to 10.2devel\n" - run(f"git checkout {MAIN_BRANCH}") - finally: - clear_env() - print("test") + update_release_return_value = update_release( + github_token=github_token, project_name="citus", project_version="10.1.0", main_branch=MAIN_BRANCH, + earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), + exec_path=test_base_path_major, is_test=True) + + run(f"git checkout {update_release_return_value.release_branch_name}") + + assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, " 10.1.0") + assert file_includes_line(test_base_path_major, CONFIGURE_IN, "AC_INIT([Citus], [10.1.0])") + assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.1.0'") + assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_STRING='Citus 10.1.0'") + assert file_includes_line(test_base_path_major, CONFIGURE, + r"\`configure' configures Citus 10.1.0 to adapt to many kinds of systems.") + assert file_includes_line(test_base_path_major, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.1.0:";;') + assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.1.0'") + assert get_last_commit_message(test_base_path_major) == "Bump citus version to 10.1.0\n" + + run(f"git checkout {update_release_return_value.upcoming_version_branch}") + + assert file_includes_line(test_base_path_major, CITUS_CONTROL, "default_version = '10.2-1'") + assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, + "-- Test downgrade to 10.1-1 from 10.2-1") + assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.1-1';") + assert count_line_in_file(test_base_path_major, MULTI_EXTENSION_OUT, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 + assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, + "-- Should be empty result since upgrade+downgrade should be a no-op") + assert count_line_in_file(test_base_path_major, MULTI_EXTENSION_OUT, + "SELECT * FROM print_extension_changes();") - previous_print_extension_changes == 2 + assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, + "-- Snapshot of state at 10.2-1") + assert file_includes_line(test_base_path_major, MULTI_EXTENSION_OUT, " 10.2devel") + + assert count_line_in_file(test_base_path_major, MULTI_EXTENSION_SQL, + "ALTER EXTENSION citus UPDATE TO '10.2-1';") == 2 + assert file_includes_line(test_base_path_major, CONFIG_PY, "MASTER_VERSION = '10.2'") + assert file_includes_line(test_base_path_major, CONFIGURE_IN, "AC_INIT([Citus], [10.2devel])") + assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_STRING='Citus 10.2devel'") + assert file_includes_line(test_base_path_major, CONFIGURE, + r"\`configure' configures Citus 10.2devel to adapt to many kinds of systems.") + assert file_includes_line(test_base_path_major, CONFIGURE, + ' short | recursive ) echo "Configuration of Citus 10.2devel:";;') + assert file_includes_line(test_base_path_major, CONFIGURE, "PACKAGE_VERSION='10.2devel'") + assert os.path.exists(f"{test_base_path_major}/{update_release_return_value.upgrade_path_sql_file}") + assert os.path.exists(f"{test_base_path_major}/{update_release_return_value.downgrade_path_sql_file}") + assert get_last_commit_message(test_base_path_major) == 'Bump citus version to 10.2devel\n' + run(f"git checkout {MAIN_BRANCH}") def test_patch_release(): - initialize_env() - os.chdir(TEST_CHECKOUT_DIR) + test_base_path_patch = initialize_env() + resources_to_be_deleted.append(test_base_path_patch) + os.chdir(test_base_path_patch) try: - update_release(github_token=github_token, project_name="citus", project_version="10.2.0", - main_branch=MAIN_BRANCH, - earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True) - + project_version = "10.0.4" + schema_version = "10.1-5" update_release( - github_token=github_token, project_name="citus", project_version="10.2.1", + github_token=github_token, project_name="citus", project_version=project_version, main_branch=MAIN_BRANCH, earliest_pr_date=datetime.strptime('2021.03.25 00:00', '%Y.%m.%d %H:%M'), - exec_path=TEST_BASE_PATH, is_test=True,schema_version="10.2-2") - assert file_includes_line(TEST_BASE_PATH, MULTI_EXTENSION_OUT, " 10.2.1") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE_IN, "AC_INIT([Citus], [10.2.1])") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_STRING='Citus 10.2.1'") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - r"\`configure' configures Citus 10.2.1 to adapt to many kinds of systems.") - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, - ' short | recursive ) echo "Configuration of Citus 10.2.1:";;') - assert file_includes_line(TEST_BASE_PATH, CONFIGURE, "PACKAGE_VERSION='10.2.1'") - assert file_includes_line(TEST_BASE_PATH, CITUS_CONTROL, "default_version = '10.2-2'") + exec_path=test_base_path_patch, is_test=True, schema_version=schema_version) + assert file_includes_line(test_base_path_patch, MULTI_EXTENSION_OUT, f" {project_version}") + assert file_includes_line(test_base_path_patch, CONFIGURE_IN, f"AC_INIT([Citus], [{project_version}])") + assert file_includes_line(test_base_path_patch, CONFIGURE, f"PACKAGE_VERSION='{project_version}'") + assert file_includes_line(test_base_path_patch, CONFIGURE, f"PACKAGE_STRING='Citus {project_version}'") + assert file_includes_line(test_base_path_patch, CONFIGURE, + rf"\`configure' configures Citus {project_version} to adapt to many kinds of systems.") + assert file_includes_line(test_base_path_patch, CONFIGURE, + f' short | recursive ) echo "Configuration of Citus {project_version}:";;') + assert file_includes_line(test_base_path_patch, CONFIGURE, f"PACKAGE_VERSION='{project_version}'") + assert file_includes_line(test_base_path_patch, CITUS_CONTROL, f"default_version = '{schema_version}'") run(f"git checkout {MAIN_BRANCH}") - finally: - clear_env() - - -def clear_env(): - if os.path.exists(f"../{TEST_CHECKOUT_DIR}"): - os.chdir("..") - run(f"sudo rm -rf {TEST_CHECKOUT_DIR}") + for path in resources_to_be_deleted: + run(f"sudo rm -rf {path}") From db1b32e5b16c2468a0b0db24344443ecb72518fa Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 9 Jun 2021 13:02:38 +0300 Subject: [PATCH 099/106] * Remove unnecessary imports * Add comment for resource_to_be_deleted array --- packaging_automation/common_tool_methods.py | 2 ++ packaging_automation/tests/test_prepare_release.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 841a2384..25c461fe 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -15,6 +15,8 @@ BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX = "\.\d{1,3}" +# When using GitPython library Repo objects should be closed to be able to delete cloned sources +# referenced by Repo objects. References are stored below array to be able to close all resources after code execution referenced_repos:List[Repo] = [] diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 335688b0..6bc5a968 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -1,10 +1,8 @@ import os import uuid from datetime import datetime -import time import pathlib2 -import shutil from ..common_tool_methods import (file_includes_line, count_line_in_file, run, get_last_commit_message, remove_cloned_code) From 5fbb9bcc7ad202350951cf3b9e7c5a2c00e62ae3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 10 Jun 2021 15:23:10 +0300 Subject: [PATCH 100/106] * Add comment on append line and prepend line functions * Remove does prefixes from branch check functions * Fix comments on repo references --- packaging_automation/common_tool_methods.py | 16 +++++++++----- packaging_automation/prepare_release.py | 8 +++---- .../tests/test_common_tool_methods.py | 22 +++++++++---------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 25c461fe..7d2442ee 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -16,7 +16,8 @@ PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX = "\.\d{1,3}" # When using GitPython library Repo objects should be closed to be able to delete cloned sources -# referenced by Repo objects. References are stored below array to be able to close all resources after code execution +# referenced by Repo objects.References are stored in below array to be able to close +# all resources after the code execution. referenced_repos:List[Repo] = [] @@ -82,7 +83,7 @@ def cherry_pick_prs(prs: List[PullRequest.PullRequest]): commits = pr.get_commits() for single_commit in commits: if not is_merge_commit(single_commit): - cp_result = run(f"git cherry-pick {single_commit.commit.sha}") + cp_result = run(f"git cherry-pick -x {single_commit.commit.sha}") print( f"Cherry pick result for PR no {pr.number} and commit sha {single_commit.commit.sha}: {cp_result}") @@ -205,6 +206,9 @@ def append_line_in_file(file: str, match_regex: str, append_str: str) -> bool: if line_number + 1 < len(lines): copy_lines[appended_line_index + 1] = append_str + # Since line is added after matched string, shift index start with line_number+1 + # increment of appended_line_index is 2 since copy_lines appended_line_index+1 includes + # append_str lines_to_be_shifted = lines[line_number + 1:] copy_lines = copy_lines[0:appended_line_index + 2] + lines_to_be_shifted else: @@ -228,6 +232,8 @@ def prepend_line_in_file(file: str, match_regex: str, append_str: str) -> bool: if re.match(match_regex, line.strip()): has_match = True copy_lines[prepended_line_index] = append_str + # Since line is added before matched string shift index start with line_number + # increment of prepend_line_index is 1 line after prepended_line_index should be shifted lines_to_be_shifted = lines[line_number:] copy_lines = copy_lines[0:prepended_line_index + 1] + lines_to_be_shifted prepended_line_index = prepended_line_index + 1 @@ -243,7 +249,7 @@ def get_current_branch(working_dir: str) -> str: return repo.active_branch -def does_remote_branch_exist(branch_name: str, working_dir: str) -> bool: +def remote_branch_exists(branch_name: str, working_dir: str) -> bool: repo = get_new_repo(working_dir) for rp in repo.references: if rp.name.endswith(f"/{branch_name}"): @@ -251,7 +257,7 @@ def does_remote_branch_exist(branch_name: str, working_dir: str) -> bool: return False -def does_local_branch_exist(branch_name: str, working_dir: str) -> bool: +def local_branch_exists(branch_name: str, working_dir: str) -> bool: repo = get_new_repo(working_dir) for rp in repo.branches: if rp.name == branch_name: @@ -260,7 +266,7 @@ def does_local_branch_exist(branch_name: str, working_dir: str) -> bool: def does_branch_exist(branch_name: str, working_dir: str) -> bool: - return does_local_branch_exist(branch_name, working_dir) or does_remote_branch_exist(branch_name, working_dir) + return local_branch_exists(branch_name, working_dir) or remote_branch_exists(branch_name, working_dir) def get_template_environment(template_dir: str) -> Environment: diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index 2092da90..ca86ad78 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -13,8 +13,8 @@ get_prs_for_patch_release, filter_prs_by_label, cherry_pick_prs, run, replace_line_in_file, get_current_branch, find_nth_matching_line_and_line_number, get_minor_version, get_patch_version_regex, - does_remote_branch_exist, does_local_branch_exist, prepend_line_in_file, - get_template_environment, get_upcoming_minor_version,remove_cloned_code) + remote_branch_exists, local_branch_exists, prepend_line_in_file, + get_template_environment, get_upcoming_minor_version, remove_cloned_code) from .common_validations import (CITUS_MINOR_VERSION_PATTERN, CITUS_PATCH_VERSION_PATTERN, is_version) MULTI_EXTENSION_SQL = "src/test/regress/sql/multi_extension.sql" @@ -192,8 +192,8 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar if patchReleaseParams.is_test: non_test_release_branch = patchReleaseParams.release_branch_name.rstrip("-test") - release_branch_exist = does_remote_branch_exist(non_test_release_branch, os.getcwd()) - test_release_branch_exist = does_local_branch_exist(patchReleaseParams.release_branch_name, os.getcwd()) + release_branch_exist = remote_branch_exists(non_test_release_branch, os.getcwd()) + test_release_branch_exist = local_branch_exists(patchReleaseParams.release_branch_name, os.getcwd()) if release_branch_exist: run(f"git checkout {non_test_release_branch}") diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 21347c82..aee6253b 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -11,10 +11,10 @@ from ..common_tool_methods import ( find_nth_occurrence_position, is_major_release, str_array_to_str, run, remove_text_with_parenthesis, get_version_details, - replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label,get_upcoming_minor_version, + replace_line_in_file, get_prs_for_patch_release, filter_prs_by_label, get_upcoming_minor_version, get_project_version_from_tag_name, find_nth_matching_line_and_line_number, get_minor_version, - get_patch_version_regex, append_line_in_file, prepend_line_in_file, does_remote_branch_exist, get_current_branch, - does_local_branch_exist,get_last_commit_message) + get_patch_version_regex, append_line_in_file, prepend_line_in_file, remote_branch_exists, get_current_branch, + local_branch_exists, get_last_commit_message) GITHUB_TOKEN = os.getenv("GH_TOKEN") TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -105,21 +105,21 @@ def test_getprs_with_backlog_label(self): self.assertEqual(1, len(prs_backlog)) self.assertEqual(4746, prs_backlog[0].number) - def test_does_local_branch_exist(self): + def test_local_branch_exists(self): current_branch_name = get_current_branch(os.getcwd()) branch_name = "develop-local-test" - self.assertTrue(does_remote_branch_exist("develop", os.getcwd())) - self.assertFalse(does_remote_branch_exist("develop2", os.getcwd())) + self.assertTrue(remote_branch_exists("develop", os.getcwd())) + self.assertFalse(remote_branch_exists("develop2", os.getcwd())) try: run(f"git checkout -b {branch_name}") - self.assertTrue(does_local_branch_exist(branch_name, os.getcwd())) + self.assertTrue(local_branch_exists(branch_name, os.getcwd())) run(f"git checkout {current_branch_name} ") finally: run(f"git branch -D {branch_name}") - self.assertFalse(does_remote_branch_exist("develop_test", os.getcwd())) + self.assertFalse(remote_branch_exists("develop_test", os.getcwd())) - def test_does_remote_branch_exist(self): + def test_remote_branch_exists(self): current_branch_name = get_current_branch(os.getcwd()) branch_name = "develop-remote-test" try: @@ -132,8 +132,8 @@ def test_does_remote_branch_exist(self): run(f"git push --set-upstream origin {branch_name}") run(f"git checkout develop") run(f"git branch -D {branch_name}") - self.assertTrue(does_remote_branch_exist(branch_name, os.getcwd())) - self.assertFalse(does_remote_branch_exist(f"{branch_name}{uuid.uuid4()}", os.getcwd())) + self.assertTrue(remote_branch_exists(branch_name, os.getcwd())) + self.assertFalse(remote_branch_exists(f"{branch_name}{uuid.uuid4()}", os.getcwd())) finally: run(f"git checkout {current_branch_name} ") # run(f"git branch -D {branch_name}") From 1d4ec86c0a5aa411c26d3ba11112b28126b881df Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 10 Jun 2021 17:13:19 +0300 Subject: [PATCH 101/106] Fix configure.in project name for citus enterprise --- packaging_automation/prepare_release.py | 30 +++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/packaging_automation/prepare_release.py b/packaging_automation/prepare_release.py index ca86ad78..d3d8a19b 100644 --- a/packaging_automation/prepare_release.py +++ b/packaging_automation/prepare_release.py @@ -49,13 +49,13 @@ MULTI_EXT_OUT_TEMPLATE_FILE = "multi_extension_out_prepare_release.tmpl" MULTI_EXT_SQL_TEMPLATE_FILE = "multi_extension_sql_prepare_release.tmpl" - -class SupportedGithubRepos(Enum): - CITUS = 1 - CITUS_ENTERPRISE = 2 - - -default_project_branches = {"citus": "master", "citus-enterprise": "enterprise-master"} +repo_details = { + "citus": { + "configure-in-str": "Citus", + "branch": "master"}, + "citus-enterprise": { + "configure-in-str": "Citus Enterprise", + "branch": "enterprise master"}} @dataclass @@ -205,7 +205,8 @@ def prepare_release_branch_for_patch_release(patchReleaseParams: PatchReleasePar else: checkout_branch(patchReleaseParams.release_branch_name, patchReleaseParams.is_test) # change version info in configure.in file - update_version_in_configure_in(patchReleaseParams.configure_in_path, patchReleaseParams.project_version) + update_version_in_configure_in(patchReleaseParams.project_name, patchReleaseParams.configure_in_path, + patchReleaseParams.project_version) # execute "auto-conf " execute_autoconf_f() # change version info in multi_extension.out @@ -237,7 +238,8 @@ def prepare_upcoming_version_branch(upcoming_params: UpcomingVersionBranchParams # create master-update-version-$curtime branch create_and_checkout_branch(upcoming_params.upcoming_version_branch) # update version info with upcoming version on configure.in - update_version_in_configure_in(upcoming_params.configure_in_path, upcoming_params.upcoming_devel_version) + update_version_in_configure_in(upcoming_params.project_name, upcoming_params.configure_in_path, + upcoming_params.upcoming_devel_version) # update version info with upcoming version on config.py update_version_with_upcoming_version_in_config_py(upcoming_params.config_py_path, upcoming_params.upcoming_minor_version) @@ -292,7 +294,8 @@ def prepare_release_branch_for_major_release(majorReleaseParams: MajorReleasePar # create release branch in release-X.Y format create_and_checkout_branch(majorReleaseParams.release_branch_name) # change version info in configure.in file - update_version_in_configure_in(majorReleaseParams.configure_in_path, majorReleaseParams.project_version) + update_version_in_configure_in(majorReleaseParams.project_name, majorReleaseParams.configure_in_path, + majorReleaseParams.project_version) # execute "autoconf -f" execute_autoconf_f() # change version info in multi_extension.out @@ -445,10 +448,10 @@ def execute_autoconf_f(): print(f"### Done autoconf -f executed. ###") -def update_version_in_configure_in(configure_in_path, project_version): +def update_version_in_configure_in(project_name,configure_in_path, project_version): print(f"### Updating version on file {configure_in_path}... ###") if not replace_line_in_file(configure_in_path, CONFIGURE_IN_SEARCH_PATTERN, - f"AC_INIT([Citus], [{project_version}])"): + f"AC_INIT([{repo_details[project_name]['configure-in-str']}], [{project_version}])"): raise ValueError(f"{configure_in_path} does not have match for version") print(f"### Done {configure_in_path} file is updated with project version {project_version}. ###") @@ -557,7 +560,7 @@ def validate_parameters(major_release_flag: bool): initialize_env(execution_path, arguments.prj_name) is_cherry_pick_enabled = arguments.cherry_pick_enabled - main_branch = arguments.main_branch if arguments.main_branch else default_project_branches[arguments.prj_name] + main_branch = arguments.main_branch if arguments.main_branch else repo_details[arguments.prj_name]["branch"] print(f"Using main branch {main_branch} for the repo {arguments.prj_name}.") os.chdir(execution_path) print(f"Executing in path {execution_path}") @@ -575,4 +578,3 @@ def validate_parameters(major_release_flag: bool): finally: if not is_test: remove_cloned_code(execution_path) - From 2abe57036874ee3ce0b3a5e0c48a90bc46112593 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 10 Jun 2021 18:30:06 +0300 Subject: [PATCH 102/106] Add readme for prepare-release script --- packaging_automation/README.md | 79 ++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 packaging_automation/README.md diff --git a/packaging_automation/README.md b/packaging_automation/README.md new file mode 100644 index 00000000..c981830b --- /dev/null +++ b/packaging_automation/README.md @@ -0,0 +1,79 @@ +# **Prepare Release Usage** + +prepare-release.py script performs the pre-packaging configurations in citus/citus-enterprise projects. + +## Installation + +Before using script, you need to make sure that Python > 3.8 is installed in your system. + +### Clone Tools Repository + +git clone https://github.com/citusdata/tools.git + +Enter 'tools' directory + +``` console +cd tools +``` + +### Install Required Python Libraries + +Verify pip installation + +``` console +python -m pip --version +``` +Output should be like following + +``` console +pip 21.1.2 from /home/vagrant/.local/lib/python3.8/site-packages/pip (python 3.8) +``` + +If you get error, you should first install pip +``` console +sudo apt install python3-pip +``` +Install the required libraries to execute the script +``` console +python -m pip install -r packaging_automation/requirements.txt +``` +If all the steps above completed successfully , you are ready for script execution + +## Script Usage + +Script can be used for either major release (i.e. third digit of release is '0' e.g. 10.1.0) or +patch release (i.e. third digit of release is other than '0' e.g. 10.0.4). + +### Available flags + +**--gh_token:** Personal access token that is authorized to commit citus/citus-enterprise projects. (Required) + +**--prj_name:** Project to be released. Allowed values 'citus' and 'citus-enterprise (Required) + +**--prj_ver:** Upcoming version to be used for release. should include three level of digits separated by dots, e.g: 10.0.1 +(Required) + +**--main-branch:** Branch to be used as base to be used for configuration changes. There is no need for base scenario. +This flag can be used for testing purposes. If not used, default branch value is used; i.e. for 'citus' 'master, for 'citus-enterprise' 'enterprise-master' + +**--is_test:** If used, branches would not be pushed remote repository and created release branches would be prefixed with 'test'. Default value is False + +**--cherry_pick_enabled:** Available only for patch release. If used, --earliest_pr_date flag also should be used.Gets all PR's with 'backport' label created after earliest_pr_date + +**--earliest_pr_date:** Used with --cherry-pick-enabled flag. Date format is 'Y.m.d' e.g 2012.01.21. PR's merged after this date would be listed and cherry-picked. + +**--schema-version:** Available only for patch release. If used, schema version in citus.control file would be updated. + +###Example Usage + +####Major +``` console +python -m packaging_automation.prepare_release --gh_token --prj_name citus --prj_ver 10.1.0 +``` +#### Patch +``` console +python -m packaging_automation.prepare_release --gh_token --prj_name citus-enterprise --prj_ver 10.0.4 --schema_version 10.0-5 +``` + + + From f91c2dca40195f9dcd0273772bacb710dad16617 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 10 Jun 2021 18:32:42 +0300 Subject: [PATCH 103/106] Remove sudo from rm statement --- packaging_automation/common_tool_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 7d2442ee..41e33efa 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -285,7 +285,7 @@ def remove_cloned_code(exec_path: str): if os.path.exists(f"{exec_path}/.git"): run(f"chmod -R 777 {exec_path}/.git") try: - run(f"sudo rm -rf {exec_path}") + run(f"rm -rf {exec_path}") print("Done. Code deleted successfully.") except: print(f"Some files could not be deleted in directory {exec_path}. " From 25ff7c1e1c5349a60ab2490d2a3187488f7fadcf Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 11 Jun 2021 08:10:04 +0300 Subject: [PATCH 104/106] Remove does prefix from does_branch_exist --- packaging_automation/common_tool_methods.py | 2 +- packaging_automation/tests/test_common_tool_methods.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 41e33efa..3a7c15c2 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -265,7 +265,7 @@ def local_branch_exists(branch_name: str, working_dir: str) -> bool: return False -def does_branch_exist(branch_name: str, working_dir: str) -> bool: +def branch_exists(branch_name: str, working_dir: str) -> bool: return local_branch_exists(branch_name, working_dir) or remote_branch_exists(branch_name, working_dir) diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index aee6253b..41e8f70d 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -1,12 +1,11 @@ import os import unittest import uuid +from datetime import datetime from shutil import copyfile import pathlib2 from github import Github -from datetime import datetime -import uuid from ..common_tool_methods import ( find_nth_occurrence_position, is_major_release, From 64aec37ec9a7ec64e384b7fa60f1474a06b740a0 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 11 Jun 2021 12:55:46 +0300 Subject: [PATCH 105/106] Fix parameter names in README --- packaging_automation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/README.md b/packaging_automation/README.md index c981830b..5cd3c7cd 100644 --- a/packaging_automation/README.md +++ b/packaging_automation/README.md @@ -53,7 +53,7 @@ patch release (i.e. third digit of release is other than '0' e.g. 10.0.4). **--prj_ver:** Upcoming version to be used for release. should include three level of digits separated by dots, e.g: 10.0.1 (Required) -**--main-branch:** Branch to be used as base to be used for configuration changes. There is no need for base scenario. +**--main_branch:** Branch to be used as base to be used for configuration changes. There is no need for base scenario. This flag can be used for testing purposes. If not used, default branch value is used; i.e. for 'citus' 'master, for 'citus-enterprise' 'enterprise-master' **--is_test:** If used, branches would not be pushed remote repository and created release branches would be prefixed with 'test'. Default value is False @@ -62,7 +62,7 @@ This flag can be used for testing purposes. If not used, default branch value is **--earliest_pr_date:** Used with --cherry-pick-enabled flag. Date format is 'Y.m.d' e.g 2012.01.21. PR's merged after this date would be listed and cherry-picked. -**--schema-version:** Available only for patch release. If used, schema version in citus.control file would be updated. +**--schema_version:** Available only for patch release. If used, schema version in citus.control file would be updated. ###Example Usage From c53aa9cccdcbb6b153996acdb35eda08d395097e Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 11 Jun 2021 13:00:48 +0300 Subject: [PATCH 106/106] Remove unnecessary files --- test1e58b070-b0bb-4f78-9ae4-3672bd131097 | 1 - test413c9add-f28a-417c-8fbc-ff20264626f5 | 1 - testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 | 1 - 3 files changed, 3 deletions(-) delete mode 100644 test1e58b070-b0bb-4f78-9ae4-3672bd131097 delete mode 100644 test413c9add-f28a-417c-8fbc-ff20264626f5 delete mode 100644 testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 diff --git a/test1e58b070-b0bb-4f78-9ae4-3672bd131097 b/test1e58b070-b0bb-4f78-9ae4-3672bd131097 deleted file mode 100644 index 3f3f005b..00000000 --- a/test1e58b070-b0bb-4f78-9ae4-3672bd131097 +++ /dev/null @@ -1 +0,0 @@ -Test content \ No newline at end of file diff --git a/test413c9add-f28a-417c-8fbc-ff20264626f5 b/test413c9add-f28a-417c-8fbc-ff20264626f5 deleted file mode 100644 index 3f3f005b..00000000 --- a/test413c9add-f28a-417c-8fbc-ff20264626f5 +++ /dev/null @@ -1 +0,0 @@ -Test content \ No newline at end of file diff --git a/testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 b/testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 deleted file mode 100644 index 3f3f005b..00000000 --- a/testdc1b03cb-b9d6-4bf8-a53e-a4b0e990a111 +++ /dev/null @@ -1 +0,0 @@ -Test content \ No newline at end of file