swrup/mte
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Browse source
This commit is contained in:
swrup 2026-02-10 07:59:24 +01:00
parent cbfcbdd046
commit b3cbdb3081
2 changed files with 593 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

489
dbinit_sql/init.sql Normal file
View file

@ -0,0 +1,489 @@
-- LICENSE AND COPYRIGHT
--
-- Copyright (C) 2010 Hubert depesz Lubaczewski
--
-- This program is distributed under the (Revised) BSD License:
-- L<http://www.opensource.org/licenses/bsd-license.php>
--
-- Redistribution and use in source and binary forms, with or without
-- modification, are permitted provided that the following conditions
-- are met:
--
-- * Redistributions of source code must retain the above copyright
-- notice, this list of conditions and the following disclaimer.
--
-- * Redistributions in binary form must reproduce the above copyright
-- notice, this list of conditions and the following disclaimer in the
-- documentation and/or other materials provided with the distribution.
--
-- * Neither the name of Hubert depesz Lubaczewski's Organization
-- nor the names of its contributors may be used to endorse or
-- promote products derived from this software without specific
-- prior written permission.
--
-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--
-- Code origin: https://gitlab.com/depesz/Versioning/blob/master/install.versioning.sql
--
--
-- # NAME
--
-- **Versioning** - simplistic take on tracking and applying changes to databases.
--
-- # DESCRIPTION
--
-- This project strives to provide simple way to manage changes to
-- database.
--
-- Instead of making changes on development server, then finding
-- differences between production and development, deciding which ones
-- should be installed on production, and finding a way to install them -
-- you start with writing diffs themselves!
--
-- # INSTALLATION
--
-- To install versioning simply run install.versioning.sql in your database
-- (all of them: production, stage, test, devel, ...).
--
-- # USAGE
--
-- In your files with patches to database, put whole logic in single
-- transaction, and use \_v.\* functions - usually \_v.register_patch() at
-- least to make sure everything is OK.
--
-- For example. Let's assume you have patch files:
--
-- ## 0001.sql:
--
-- ```
-- create table users (id serial primary key, username text);
-- ```
--
-- ## 0002.sql:
--
-- ```
-- insert into users (username) values ('depesz');
-- ```
-- To change it to use versioning you would change the files, to this
-- state:
--
-- 0000.sql:
--
-- ```
-- BEGIN;
-- select _v.register_patch('000-base', NULL, NULL);
-- create table users (id serial primary key, username text);
-- COMMIT;
-- ```
--
-- ## 0002.sql:
--
-- ```
-- BEGIN;
-- select _v.register_patch('001-users', ARRAY['000-base'], NULL);
-- insert into users (username) values ('depesz');
-- COMMIT;
-- ```
--
-- This will make sure that patch 001-users can only be applied after
-- 000-base.
--
-- # AVAILABLE FUNCTIONS
--
-- ## \_v.register_patch( TEXT )
--
-- Registers named patch, or dies if it is already registered.
--
-- Returns integer which is id of patch in \_v.patches table - only if it
-- succeeded.
--
-- ## \_v.register_patch( TEXT, TEXT[] )
--
-- Same as \_v.register_patch( TEXT ), but checks is all given patches (given as
-- array in second argument) are already registered.
--
-- ## \_v.register_patch( TEXT, TEXT[], TEXT[] )
--
-- Same as \_v.register_patch( TEXT, TEXT[] ), but also checks if there are no conflicts with preexisting patches.
--
-- Third argument is array of names of patches that conflict with current one. So
-- if any of them is installed - register_patch will error out.
--
-- ## \_v.unregister_patch( TEXT )
--
-- Removes information about given patch from the versioning data.
--
-- It doesn't remove objects that were created by this patch - just removes
-- metainformation.
--
-- ## \_v.assert_user_is_superuser()
--
-- Make sure that current patch is being loaded by superuser.
--
-- If it's not - it will raise exception, and break transaction.
--
-- ## \_v.assert_user_is_not_superuser()
--
-- Make sure that current patch is not being loaded by superuser.
--
-- If it is - it will raise exception, and break transaction.
--
-- ## \_v.assert_user_is_one_of(TEXT, TEXT, ... )
--
-- Make sure that current patch is being loaded by one of listed users.
--
-- If ```current_user``` is not listed as one of arguments - function will raise
-- exception and break the transaction.
BEGIN;
-- This file adds versioning support to database it will be loaded to.
-- It requires that PL/pgSQL is already loaded - will raise exception otherwise.
-- All versioning "stuff" (tables, functions) is in "_v" schema.
-- All functions are defined as 'RETURNS SETOF INT4' to be able to make them to RETURN literally nothing (0 rows).
-- >> RETURNS VOID<< IS similar, but it still outputs "empty line" in psql when calling
CREATE SCHEMA IF NOT EXISTS _v;
COMMENT ON SCHEMA _v IS 'Schema for versioning data and functionality.';
CREATE TABLE IF NOT EXISTS _v.patches (
patch_name TEXT PRIMARY KEY,
applied_tsz TIMESTAMPTZ NOT NULL DEFAULT now(),
applied_by TEXT NOT NULL,
requires TEXT[],
conflicts TEXT[]
);
COMMENT ON TABLE _v.patches IS 'Contains information about what patches are currently applied on database.';
COMMENT ON COLUMN _v.patches.patch_name IS 'Name of patch, has to be unique for every patch.';
COMMENT ON COLUMN _v.patches.applied_tsz IS 'When the patch was applied.';
COMMENT ON COLUMN _v.patches.applied_by IS 'Who applied this patch (PostgreSQL username)';
COMMENT ON COLUMN _v.patches.requires IS 'List of patches that are required for given patch.';
COMMENT ON COLUMN _v.patches.conflicts IS 'List of patches that conflict with given patch.';
CREATE OR REPLACE FUNCTION _v.register_patch( IN in_patch_name TEXT, IN in_requirements TEXT[], in_conflicts TEXT[], OUT versioning INT4 ) RETURNS setof INT4 AS $$
DECLARE
t_text TEXT;
t_text_a TEXT[];
i INT4;
BEGIN
-- Thanks to this we know only one patch will be applied at a time
LOCK TABLE _v.patches IN EXCLUSIVE MODE;
SELECT patch_name INTO t_text FROM _v.patches WHERE patch_name = in_patch_name;
IF FOUND THEN
RAISE EXCEPTION 'Patch % is already applied!', in_patch_name;
END IF;
t_text_a := ARRAY( SELECT patch_name FROM _v.patches WHERE patch_name = any( in_conflicts ) );
IF array_upper( t_text_a, 1 ) IS NOT NULL THEN
RAISE EXCEPTION 'Versioning patches conflict. Conflicting patche(s) installed: %.', array_to_string( t_text_a, ', ' );
END IF;
IF array_upper( in_requirements, 1 ) IS NOT NULL THEN
t_text_a := '{}';
FOR i IN array_lower( in_requirements, 1 ) .. array_upper( in_requirements, 1 ) LOOP
SELECT patch_name INTO t_text FROM _v.patches WHERE patch_name = in_requirements[i];
IF NOT FOUND THEN
t_text_a := t_text_a || in_requirements[i];
END IF;
END LOOP;
IF array_upper( t_text_a, 1 ) IS NOT NULL THEN
RAISE EXCEPTION 'Missing prerequisite(s): %.', array_to_string( t_text_a, ', ' );
END IF;
END IF;
INSERT INTO _v.patches (patch_name, applied_tsz, applied_by, requires, conflicts ) VALUES ( in_patch_name, now(), current_user, coalesce( in_requirements, '{}' ), coalesce( in_conflicts, '{}' ) );
RETURN;
END;
$$ language plpgsql;
COMMENT ON FUNCTION _v.register_patch( TEXT, TEXT[], TEXT[] ) IS 'Function to register patches in database. Raises exception if there are conflicts, prerequisites are not installed or the migration has already been installed.';
CREATE OR REPLACE FUNCTION _v.register_patch( TEXT, TEXT[] ) RETURNS setof INT4 AS $$
SELECT _v.register_patch( $1, $2, NULL );
$$ language sql;
COMMENT ON FUNCTION _v.register_patch( TEXT, TEXT[] ) IS 'Wrapper to allow registration of patches without conflicts.';
CREATE OR REPLACE FUNCTION _v.register_patch( TEXT ) RETURNS setof INT4 AS $$
SELECT _v.register_patch( $1, NULL, NULL );
$$ language sql;
COMMENT ON FUNCTION _v.register_patch( TEXT ) IS 'Wrapper to allow registration of patches without requirements and conflicts.';
CREATE OR REPLACE FUNCTION _v.unregister_patch( IN in_patch_name TEXT, OUT versioning INT4 ) RETURNS setof INT4 AS $$
DECLARE
i INT4;
t_text_a TEXT[];
BEGIN
-- Thanks to this we know only one patch will be applied at a time
LOCK TABLE _v.patches IN EXCLUSIVE MODE;
t_text_a := ARRAY( SELECT patch_name FROM _v.patches WHERE in_patch_name = ANY( requires ) );
IF array_upper( t_text_a, 1 ) IS NOT NULL THEN
RAISE EXCEPTION 'Cannot uninstall %, as it is required by: %.', in_patch_name, array_to_string( t_text_a, ', ' );
END IF;
DELETE FROM _v.patches WHERE patch_name = in_patch_name;
GET DIAGNOSTICS i = ROW_COUNT;
IF i < 1 THEN
RAISE EXCEPTION 'Patch % is not installed, so it can''t be uninstalled!', in_patch_name;
END IF;
RETURN;
END;
$$ language plpgsql;
COMMENT ON FUNCTION _v.unregister_patch( TEXT ) IS 'Function to unregister patches in database. Dies if the patch is not registered, or if unregistering it would break dependencies.';
CREATE OR REPLACE FUNCTION _v.assert_patch_is_applied( IN in_patch_name TEXT ) RETURNS TEXT as $$
DECLARE
t_text TEXT;
BEGIN
SELECT patch_name INTO t_text FROM _v.patches WHERE patch_name = in_patch_name;
IF NOT FOUND THEN
RAISE EXCEPTION 'Patch % is not applied!', in_patch_name;
END IF;
RETURN format('Patch %s is applied.', in_patch_name);
END;
$$ language plpgsql;
COMMENT ON FUNCTION _v.assert_patch_is_applied( TEXT ) IS 'Function that can be used to make sure that patch has been applied.';
CREATE OR REPLACE FUNCTION _v.assert_user_is_superuser() RETURNS TEXT as $$
DECLARE
v_super bool;
BEGIN
SELECT usesuper INTO v_super FROM pg_user WHERE usename = current_user;
IF v_super THEN
RETURN 'assert_user_is_superuser: OK';
END IF;
RAISE EXCEPTION 'Current user is not superuser - cannot continue.';
END;
$$ language plpgsql;
COMMENT ON FUNCTION _v.assert_user_is_superuser() IS 'Function that can be used to make sure that patch is being applied using superuser account.';
CREATE OR REPLACE FUNCTION _v.assert_user_is_not_superuser() RETURNS TEXT as $$
DECLARE
v_super bool;
BEGIN
SELECT usesuper INTO v_super FROM pg_user WHERE usename = current_user;
IF v_super THEN
RAISE EXCEPTION 'Current user is superuser - cannot continue.';
END IF;
RETURN 'assert_user_is_not_superuser: OK';
END;
$$ language plpgsql;
COMMENT ON FUNCTION _v.assert_user_is_not_superuser() IS 'Function that can be used to make sure that patch is being applied using normal (not superuser) account.';
CREATE OR REPLACE FUNCTION _v.assert_user_is_one_of(VARIADIC p_acceptable_users TEXT[] ) RETURNS TEXT as $$
DECLARE
BEGIN
IF current_user = any( p_acceptable_users ) THEN
RETURN 'assert_user_is_one_of: OK';
END IF;
RAISE EXCEPTION 'User is not one of: % - cannot continue.', p_acceptable_users;
END;
$$ language plpgsql;
COMMENT ON FUNCTION _v.assert_user_is_one_of(TEXT[]) IS 'Function that can be used to make sure that patch is being applied by one of defined users.';
COMMIT;
--
-- This file is part of TALER
-- Copyright (C) 2014--2022 Taler Systems SA
--
-- TALER is free software; you can redistribute it and/or modify it under the
-- terms of the GNU General Public License as published by the Free Software
-- Foundation; either version 3, or (at your option) any later version.
--
-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
--
-- You should have received a copy of the GNU General Public License along with
-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
--
BEGIN;
SELECT _v.register_patch('exchange-0001', NULL, NULL);
CREATE SCHEMA exchange;
COMMENT ON SCHEMA exchange IS 'taler-exchange data';
SET search_path TO exchange;
---------------------------------------------------------------------------
-- General procedures for DB setup
---------------------------------------------------------------------------
CREATE TABLE exchange_tables
(table_serial_id INT8 GENERATED BY DEFAULT AS IDENTITY
,name TEXT NOT NULL
,version TEXT NOT NULL
,action TEXT NOT NULL
,partitioned BOOL NOT NULL
,by_range BOOL NOT NULL
,finished BOOL NOT NULL DEFAULT(FALSE));
COMMENT ON TABLE exchange_tables
IS 'Tables of the exchange and their status';
COMMENT ON COLUMN exchange_tables.name
IS 'Base name of the table (without partition/shard)';
COMMENT ON COLUMN exchange_tables.version
IS 'Version of the DB in which the given action happened';
COMMENT ON COLUMN exchange_tables.action
IS 'Action to take on the table (e.g. create, alter, constrain, or foreign). Create is done for the master table and each partition; constrain is only for partitions or for master if there are no partitions; master only on master (takes no argument); foreign only on master if there are no partitions.';
COMMENT ON COLUMN exchange_tables.partitioned
IS 'TRUE if the table is partitioned';
COMMENT ON COLUMN exchange_tables.by_range
IS 'TRUE if the table is partitioned by range';
COMMENT ON COLUMN exchange_tables.finished
IS 'TRUE if the respective migration has been run';
CREATE INDEX exchange_tables_by_pending
ON exchange_tables (table_serial_id)
WHERE NOT finished;
COMMENT ON INDEX exchange_tables_by_pending
IS 'Used by exchange_do_create_tables';
COMMIT;
--
-- This file is part of TALER
-- Copyright (C) 2025 Taler Systems SA
--
-- TALER is free software; you can redistribute it and/or modify it under the
-- terms of the GNU General Public License as published by the Free Software
-- Foundation; either version 3, or (at your option) any later version.
--
-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
--
-- You should have received a copy of the GNU General Public License along with
-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
--
BEGIN;
SELECT _v.register_patch('exchange-0005', NULL, NULL);
SET search_path TO exchange;
-- convert all JSON-valued fields from TEXT to JSONB
CREATE FUNCTION alter_table_wire_accounts5()
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
ALTER TABLE wire_accounts
ALTER COLUMN debit_restrictions
TYPE JSONB
USING debit_restrictions::JSONB,
ALTER COLUMN credit_restrictions
TYPE JSONB
USING credit_restrictions::JSONB;
END
$$;
CREATE FUNCTION alter_table_legitimization_outcomes5()
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
ALTER TABLE legitimization_outcomes
ALTER COLUMN jproperties
TYPE JSONB
USING jproperties::JSONB,
ALTER COLUMN jnew_rules
TYPE JSONB
USING jnew_rules::JSONB;
END
$$;
CREATE FUNCTION alter_table_legitimization_measures5()
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
ALTER TABLE legitimization_measures
ALTER COLUMN jmeasures
TYPE JSONB
USING jmeasures::JSONB;
END
$$;
CREATE FUNCTION alter_table_policy_fulfillments5()
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
ALTER TABLE policy_fulfillments
ALTER COLUMN fulfillment_proof
TYPE JSONB
USING fulfillment_proof::JSONB;
END
$$;
CREATE FUNCTION alter_table_kyc_targets5()
RETURNS void
LANGUAGE plpgsql
AS $$
DECLARE
table_name TEXT DEFAULT 'kyc_targets';
BEGIN
EXECUTE FORMAT (
'ALTER TABLE ' || table_name ||
' ADD COLUMN open_time INT8 DEFAULT(NULL)'
',ADD COLUMN close_time INT8 DEFAULT(NULL);'
);
END
$$;
INSERT INTO exchange_tables
(name
,version
,action
,partitioned
,by_range)
VALUES
('wire_accounts5'
,'exchange-0005'
,'alter'
,TRUE
,FALSE),
('legitimization_outcomes5'
,'exchange-0005'
,'alter'
,TRUE
,FALSE),
('legitimization_measures5'
,'exchange-0005'
,'alter'
,TRUE
,FALSE),
('policy_fulfillments5'
,'exchange-0005'
,'alter'
,TRUE
,FALSE),
('kyc_targets5'
,'exchange-0005'
,'alter'
,TRUE
,FALSE);
COMMIT;

104
tools/dbinit.sh Executable file
View file

@ -0,0 +1,104 @@
#!/bin/bash
# TODO
# - PostgreSQL configuration file/env variable
# (host, port, username, password, ..)
# usage:
# `make create_database`
# create a `taler-exchange` database
# `make fetch_source`
# fetch all *.sql and *.sql.in files from GNU Taler exchange repository (latest commit)
# file `exchangedb.version` contains hash of latest commit that changed
# something in exchange/src/exchangedb
# `make init`
# initialize taler-exchange database (create tables, ...)
# `taler-exchange` database must already be created
#
set -e
taler_repo_url="https://git-www.taler.net/exchange.git/"
tmp_dir="/tmp/taler_exchange"
out_dir="./dbinit_sql"
out_init_sql="${out_dir}/init.sql"
out_init_drop="${out_dir}/drop.sql"
dbname="taler-exchange"
fetch() {
git clone --depth 1 $taler_repo_url $tmp_dir
#cp /tmp/exchange/src/exchangedb/*.sql $out_dir/
#cp /tmp/exchange/src/exchangedb/*.sql.in $out_dir/
}
process() {
source_dir="${tmp_dir}/src/exchangedb"
sql_in_files=(
"procedures.sql"
"exchange-0002.sql"
"exchange-0003.sql"
"exchange-0004.sql"
)
for file in "${sql_in_files[@]}"; do
echo "processing ${file}.in"
file_in="${source_dir}/${file}.in"
file_out="${source_dir}/${file}"
gcc -E -P -undef $file_in \
2>/dev/null \
| sed -e "s/--.*//" \
| awk 'NF' - \
> $file_out
done
# order is important
files=(
"versioning.sql"
"exchange-0001.sql"
"exchange-0002.sql"
"exchange-0003.sql"
"exchange-0004.sql"
"exchange-0005.sql"
"procedures.sql" )
mkdir -p "$out_dir"
echo "" > $out_init_sql
for file in "${files[@]}"; do
cat "${source_dir}/${file}" >> "$out_init_sql"
done
}
if [[ $# -eq 0 ]]; then
echo "no arguments" >&2
exit 1
fi
cmd=$1
case "$cmd" in
fetch) fetch "$@"; exit 0;;
process) process "$@"; exit 0;;
*) echo "Unknown command: $cmd" >&2; exit 1;;
esac
#clean:
# rm -f procedures.sql
# rm -f exchange-0002.sql
# rm -f exchange-0003.sql
# rm -f exchange-0004.sql
# rm $(sql_INIT)
#
## "NOTICE: function xxx() does not exist, skipping" are normal and expected
#init: gen_init
# psql --host=localhost --port=5432 --username=mte --password --dbname=$(dbname) --file=$(sql_INIT)
#
#drop:
# psql --host=localhost --port=5432 --username=mte --password --dbname=$(dbname) --file=$(sql_DROP)
#
#create_database:
# createdb --host=localhost --port=5432 --username=mte --password $(dbname)
#
#drop_database:
# dropdb --host=localhost --port=5432 --username=mte --password $(dbname)
#
#reset_database: drop_database create_database
#