diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 18c292cac25..d79c08b35a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + name: Publish on: diff --git a/distro/src/main/assembly/admin-web.xml b/distro/src/main/assembly/admin-web.xml index 08d0bef290a..ba7a840e4e3 100644 --- a/distro/src/main/assembly/admin-web.xml +++ b/distro/src/main/assembly/admin-web.xml @@ -602,6 +602,7 @@ setup.sh setup_authentication.sh set_globals.sh + pg_jdbc_util.py db_setup.py dba_script.py restrict_permissions.py diff --git a/distro/src/main/assembly/kms.xml b/distro/src/main/assembly/kms.xml index 6e52fd3273e..ba910cc1a1b 100755 --- a/distro/src/main/assembly/kms.xml +++ b/distro/src/main/assembly/kms.xml @@ -500,5 +500,10 @@ 755 + + ${project.parent.basedir}/security-admin/scripts/pg_jdbc_util.py + + 544 + diff --git a/kms/scripts/db_setup.py b/kms/scripts/db_setup.py index f29af9f8d42..b743c29ddcf 100644 --- a/kms/scripts/db_setup.py +++ b/kms/scripts/db_setup.py @@ -26,6 +26,7 @@ from datetime import date import datetime from time import gmtime, strftime +from pg_jdbc_util import build_pg_query_params globalDict = {} os_name = platform.system() @@ -353,16 +354,17 @@ def get_jisql_cmd(self, user, password, db_name): db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s -Djavax.net.ssl.trustStoreType=%s -Djavax.net.ssl.keyStoreType=%s" %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword,self.javax_net_ssl_trustStore_type,self.javax_net_ssl_keyStore_type) else: db_ssl_param="?ssl=%s" %(self.db_ssl_enabled) + db_query_param = build_pg_query_params(db_ssl_param, db_host=self.host) if is_unix: if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: jisql_cmd = "%s %s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring %s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,self.JAVA_OPTS,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.db_override_jdbc_connection_string,user, password) else: - jisql_cmd = "%s %s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,self.JAVA_OPTS,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_ssl_param,user, password) + jisql_cmd = "%s %s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN,self.JAVA_OPTS,db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_query_param,user, password) elif os_name == "WINDOWS": if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: jisql_cmd = "%s %s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring %s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN,self.JAVA_OPTS,db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.db_override_jdbc_connection_string,user, password) else: - jisql_cmd = "%s %s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN,self.JAVA_OPTS,db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_ssl_param,user, password) + jisql_cmd = "%s %s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN,self.JAVA_OPTS,db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_query_param,user, password) return jisql_cmd def check_connection(self, db_name, db_user, db_password): diff --git a/kms/scripts/dba_script.py b/kms/scripts/dba_script.py index 5d19a7d2157..e970d465559 100755 --- a/kms/scripts/dba_script.py +++ b/kms/scripts/dba_script.py @@ -25,6 +25,7 @@ from os.path import basename from subprocess import Popen,PIPE from datetime import date +from pg_jdbc_util import build_pg_query_params try: input = raw_input except NameError: pass globalDict = {} @@ -610,10 +611,11 @@ def get_jisql_cmd(self, user, password, db_name): db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) else: db_ssl_param="?ssl=%s&sslfactory=org.postgresql.ssl.NonValidatingFactory" %(self.db_ssl_enabled) + db_query_param = build_pg_query_params(db_ssl_param, db_host=self.host) if is_unix: - jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_ssl_param,user, password) + jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_query_param,user, password) elif os_name == "WINDOWS": - jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_ssl_param,user, password) + jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_query_param,user, password) return jisql_cmd def verify_user(self, root_user, db_root_password, db_user,dryMode): diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties index 8ce663b903a..694910de438 100755 --- a/kms/scripts/install.properties +++ b/kms/scripts/install.properties @@ -48,7 +48,10 @@ SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar #db_host=host:port # for DB_FLAVOR=MYSQL|POSTGRES|SQLA|MSSQL #for example: db_host=localhost:3306 #db_host=host:port:SID # for DB_FLAVOR=ORACLE #for SID example: db_host=localhost:1521:ORCL #db_host=host:port/ServiceName # for DB_FLAVOR=ORACLE #for Service example: db_host=localhost:1521/XE -#db_host=host:port:GL # for DB_FLAVOR=ORACLE #for TNSNAME example: db_host=localhost:1521:GL +#db_host=host:port/ServiceName # for DB_FLAVOR=ORACLE #for Service example: db_host=localhost:1521/XE +#db_host=host1:port1,host2:port2 # for DB_FLAVOR=POSTGRES (Patroni cluster) #for example: db_host=pg1:5432,pg2:5432 +# # targetServerType=primary is added automatically for multi-host Postgres to ensure +# # DDL/migration operations connect to the Patroni leader, not a read-only replica db_root_user=root db_root_password= db_host=localhost diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index b547b1751b5..b360fc97cba 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -578,22 +578,33 @@ update_properties() { fi if [ "${DB_FLAVOR}" == "POSTGRES" ] then + if [[ "${DB_HOST}" == *","* ]]; then + pg_target_server_type="targetServerType=primary" + else + pg_target_server_type="" + fi + if [ "${db_ssl_enabled}" == "true" ] then - if test -f $db_ssl_certificate_file; then - propertyName=ranger.ks.jpa.jdbc.url - newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslrootcert=${db_ssl_certificate_file}" - updatePropertyToFilePy $propertyName $newPropertyValue $to_file + if test -f "${db_ssl_certificate_file}"; then + pg_base_url="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslrootcert=${db_ssl_certificate_file}" + else + pg_base_url="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory" + fi + if [ -n "${pg_target_server_type}" ]; then + newPropertyValue="${pg_base_url}&${pg_target_server_type}" else - propertyName=ranger.ks.jpa.jdbc.url - newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory" - updatePropertyToFilePy $propertyName $newPropertyValue $to_file + newPropertyValue="${pg_base_url}" fi else - propertyName=ranger.ks.jpa.jdbc.url - newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}" - updatePropertyToFilePy $propertyName $newPropertyValue $to_file + if [ -n "${pg_target_server_type}" ]; then + newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}?${pg_target_server_type}" + else + newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}" + fi fi + propertyName=ranger.ks.jpa.jdbc.url + updatePropertyToFilePy $propertyName "$newPropertyValue" $to_file propertyName=ranger.ks.jpa.jdbc.dialect newPropertyValue="org.eclipse.persistence.platform.database.PostgreSQLPlatform" diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index 756509fe0a3..73bb2e4aca0 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -24,6 +24,7 @@ import socket import glob import getpass +from pg_jdbc_util import build_pg_query_params globalDict = {} os_name = platform.system() @@ -1008,16 +1009,17 @@ def get_jisql_cmd(self, user, password, db_name): db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s -Djavax.net.ssl.trustStoreType=%s -Djavax.net.ssl.keyStoreType=%s" %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword,self.javax_net_ssl_trustStore_type,self.javax_net_ssl_keyStore_type) else: db_ssl_param="?ssl=%s" %(self.db_ssl_enabled) + db_query_param = build_pg_query_params(db_ssl_param, db_host=self.host) if is_unix: if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring %s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.db_override_jdbc_connection_string, user, password) else: - jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_ssl_param,user, password) + jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_query_param,user, password) elif os_name == "WINDOWS": if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring %s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.db_override_jdbc_connection_string,user, password) else: - jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_ssl_param,user, password) + jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_query_param,user, password) return jisql_cmd def create_language_plpgsql(self,db_user, db_password, db_name): diff --git a/security-admin/scripts/dba_script.py b/security-admin/scripts/dba_script.py index 8a334de063c..8d553d6c412 100644 --- a/security-admin/scripts/dba_script.py +++ b/security-admin/scripts/dba_script.py @@ -28,6 +28,7 @@ import time try: input = raw_input except NameError: pass +from pg_jdbc_util import build_pg_query_params globalDict = {} os_name = platform.system() @@ -727,7 +728,7 @@ def writeDrymodeCmd(self, xa_db_host, audit_db_host, xa_db_root_user, xa_db_root class PostgresConf(BaseDB): # Constructor - def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type): + def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type,is_db_override_jdbc_connection_string='false',db_override_jdbc_connection_string=''): self.host = host.lower() self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN @@ -739,6 +740,8 @@ def __init__(self, host,SQL_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_require self.javax_net_ssl_keyStorePassword=javax_net_ssl_keyStorePassword self.javax_net_ssl_trustStore=javax_net_ssl_trustStore self.javax_net_ssl_trustStorePassword=javax_net_ssl_trustStorePassword + self.is_db_override_jdbc_connection_string = is_db_override_jdbc_connection_string + self.db_override_jdbc_connection_string = db_override_jdbc_connection_string def get_jisql_cmd(self, user, password, db_name): #TODO: User array for forming command @@ -755,10 +758,17 @@ def get_jisql_cmd(self, user, password, db_name): db_ssl_cert_param=" -Djavax.net.ssl.keyStore=%s -Djavax.net.ssl.keyStorePassword=%s -Djavax.net.ssl.trustStore=%s -Djavax.net.ssl.trustStorePassword=%s " %(self.javax_net_ssl_keyStore,self.javax_net_ssl_keyStorePassword,self.javax_net_ssl_trustStore,self.javax_net_ssl_trustStorePassword) else: db_ssl_param="?ssl=%s&sslfactory=org.postgresql.ssl.NonValidatingFactory" %(self.db_ssl_enabled) + db_query_param = build_pg_query_params(db_ssl_param, db_host=self.host) if is_unix: - jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_ssl_param,user, password) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring %s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.db_override_jdbc_connection_string, user, password) + else: + jisql_cmd = "%s %s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p '%s' -noheader -trim -c \;" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR,path, self.host, db_name, db_query_param,user, password) elif os_name == "WINDOWS": - jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_ssl_param,user, password) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring %s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.db_override_jdbc_connection_string,user, password) + else: + jisql_cmd = "%s %s -cp %s;%s\jisql\\lib\\* org.apache.util.sql.Jisql -driver postgresql -cstring jdbc:postgresql://%s/%s%s -u %s -p \"%s\" -noheader -trim" %(self.JAVA_BIN, db_ssl_cert_param,self.SQL_CONNECTOR_JAR, path, self.host, db_name, db_query_param,user, password) return jisql_cmd def verify_user(self, root_user, db_root_password, db_user,dryMode): @@ -1748,7 +1758,7 @@ def main(argv): elif XA_DB_FLAVOR == "POSTGRES": POSTGRES_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = PostgresConf(xa_db_host, POSTGRES_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) + xa_sqlObj = PostgresConf(xa_db_host, POSTGRES_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type,is_override_db_connection_string,db_override_jdbc_connection_string) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME,postgres_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME,postgres_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME,postgres_patches) @@ -1789,7 +1799,7 @@ def main(argv): elif AUDIT_DB_FLAVOR == "POSTGRES": POSTGRES_CONNECTOR_JAR=CONNECTOR_JAR - audit_sqlObj = PostgresConf(audit_db_host,POSTGRES_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) + audit_sqlObj = PostgresConf(audit_db_host,POSTGRES_CONNECTOR_JAR,JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type,is_override_db_connection_string,db_override_jdbc_connection_string) audit_db_file = os.path.join(RANGER_ADMIN_HOME,postgres_audit_file) elif AUDIT_DB_FLAVOR == "MSSQL": diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties index eb157f943e2..c8dd513fd8b 100644 --- a/security-admin/scripts/install.properties +++ b/security-admin/scripts/install.properties @@ -49,6 +49,9 @@ SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar #db_host=host:port # for DB_FLAVOR=MYSQL|POSTGRES|SQLA|MSSQL #for example: db_host=localhost:3306 #db_host=host:port:SID # for DB_FLAVOR=ORACLE #for SID example: db_host=localhost:1521:ORCL #db_host=host:port/ServiceName # for DB_FLAVOR=ORACLE #for Service example: db_host=localhost:1521/XE +#db_host=host1:port1,host2:port2 # for DB_FLAVOR=POSTGRES (Patroni cluster) #for example: db_host=pg1:5432,pg2:5432 +# # targetServerType=primary is added automatically for multi-host Postgres to ensure +# # DDL/migration operations connect to the Patroni leader, not a read-only replica db_root_user=root db_root_password= db_host=localhost diff --git a/security-admin/scripts/pg_jdbc_util.py b/security-admin/scripts/pg_jdbc_util.py new file mode 100644 index 00000000000..da53d59799b --- /dev/null +++ b/security-admin/scripts/pg_jdbc_util.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Helpers for PostgreSQL JDBC URL host segments and query parameters (Patroni multi-host)""" + +import argparse +import sys + +DEFAULT_PG_PORT = 5432 +TARGET_SERVER_TYPE_PRIMARY = 'targetServerType=primary' + + +def is_multi_host(db_host): + if db_host is None: + return False + return ',' in str(db_host) + + +def parse_pg_hosts(db_host, default_port=DEFAULT_PG_PORT): + """ + Parse db_host into a list of (host, port) tuples + + Supported formats: + host + host:port + host1:port1,host2:port2 + host1,host2 (default_port applied when port omitted) + """ + if db_host is None or not str(db_host).strip(): + return [] + + result = [] + for segment in str(db_host).split(','): + segment = segment.strip().lower() + if not segment: + continue + if ':' in segment: + host_part, port_part = segment.rsplit(':', 1) + if port_part.isdigit(): + result.append((host_part, int(port_part))) + else: + result.append((segment, default_port)) + else: + result.append((segment, default_port)) + return result + + +def build_pg_host_segment(host, port=None): + """ + Build the host segment for jdbc:postgresql://{segment}/{db} + + For Patroni multi-host (comma in host), returns host unchanged + For a single host without port, appends :port (default 5432) + """ + if host is None: + return '' + host = str(host).strip().lower() + if not host: + return '' + if is_multi_host(host): + return host + port_str = str(port if port is not None else DEFAULT_PG_PORT).strip() + if ':' in host: + return host + return '%s:%s' % (host, port_str) + + +def build_pg_query_params(existing_ssl_param, is_multi_host_flag=None, db_host=None): + """ + Merge SSL/query parameters with targetServerType=primary for multi-host URLs + + existing_ssl_param is typically empty or starts with '?' (e.g. ?ssl=true&sslmode=...) + When connecting to multiple hosts, PostgreSQL JDBC may pick a read-only replica; + targetServerType=primary directs DDL/migration traffic to the Patroni leader + """ + if is_multi_host_flag is None: + is_multi_host_flag = is_multi_host(db_host) + + param = existing_ssl_param if existing_ssl_param is not None else '' + if not is_multi_host_flag: + return param + if TARGET_SERVER_TYPE_PRIMARY in param: + return param + if not param: + return '?' + TARGET_SERVER_TYPE_PRIMARY + if param.startswith('?'): + return param + '&' + TARGET_SERVER_TYPE_PRIMARY + return '?' + param + '&' + TARGET_SERVER_TYPE_PRIMARY + + +def build_pg_jdbc_url(host_segment, db_name, query_suffix=''): + host_segment = host_segment if host_segment is not None else '' + db_name = db_name if db_name is not None else '' + query_suffix = query_suffix if query_suffix is not None else '' + return 'jdbc:postgresql://%s/%s%s' % (host_segment, db_name, query_suffix) + + +def _main(): + parser = argparse.ArgumentParser(description='PostgreSQL JDBC URL helpers for Ranger install scripts') + sub = parser.add_subparsers(dest='command', required=True) + + query_parser = sub.add_parser('build-query-params', help='Build JDBC query suffix for SSL + Patroni') + query_parser.add_argument('--ssl-param', default='', help='Existing query string (e.g. ?ssl=true)') + query_parser.add_argument('--db-host', default='', help='db_host value to detect multi-host') + + host_parser = sub.add_parser('build-host-segment', help='Build JDBC host segment from host and port') + host_parser.add_argument('--host', required=True) + host_parser.add_argument('--port', default=str(DEFAULT_PG_PORT)) + + args = parser.parse_args() + if args.command == 'build-query-params': + multi = is_multi_host(args.db_host) + sys.stdout.write(build_pg_query_params(args.ssl_param, is_multi_host_flag=multi)) + elif args.command == 'build-host-segment': + sys.stdout.write(build_pg_host_segment(args.host, args.port)) + return 0 + + +if __name__ == '__main__': + sys.exit(_main()) diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index 1fef99ee452..d3946acc1bf 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -706,23 +706,33 @@ update_properties() { fi if [ "${DB_FLAVOR}" == "POSTGRES" ] then + if [[ "${DB_HOST}" == *","* ]]; then + pg_target_server_type="targetServerType=primary" + else + pg_target_server_type="" + fi if [ "${db_ssl_enabled}" == "true" ] then - if test -f $db_ssl_certificate_file; then - propertyName=ranger.jpa.jdbc.url - newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslrootcert=${db_ssl_certificate_file}" - updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger + if test -f "${db_ssl_certificate_file}"; then + pg_base_url="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslrootcert=${db_ssl_certificate_file}" else - propertyName=ranger.jpa.jdbc.url - newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory" - updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger + pg_base_url="jdbc:postgresql://${DB_HOST}/${db_name}?ssl=true&sslmode=verify-full&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory" + fi + if [ -n "${pg_target_server_type}" ]; then + newPropertyValue="${pg_base_url}&${pg_target_server_type}" + else + newPropertyValue="${pg_base_url}" fi else - propertyName=ranger.jpa.jdbc.url - newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}" - updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger + if [ -n "${pg_target_server_type}" ]; then + newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}?${pg_target_server_type}" + else + newPropertyValue="jdbc:postgresql://${DB_HOST}/${db_name}" + fi fi + propertyName=ranger.jpa.jdbc.url + updatePropertyToFilePy $propertyName "$newPropertyValue" $to_file_ranger propertyName=ranger.jpa.jdbc.dialect newPropertyValue="org.eclipse.persistence.platform.database.PostgreSQLPlatform" diff --git a/security-admin/src/bin/ranger_install.py b/security-admin/src/bin/ranger_install.py index 39b9d1f81e5..ad4176e4603 100644 --- a/security-admin/src/bin/ranger_install.py +++ b/security-admin/src/bin/ranger_install.py @@ -35,6 +35,25 @@ import pprint from subprocess import Popen,PIPE +try: + sys.path.insert(0, os.getenv('RANGER_ADMIN_HOME', os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'scripts'))) + from pg_jdbc_util import build_pg_host_segment, build_pg_query_params +except ImportError: + def build_pg_host_segment(host, port=None): + if not host: + return '' + host = str(host).strip() + if ',' in host or ':' in host: + return host + return '%s:%s' % (host, str(port or 5432)) + def build_pg_query_params(existing_ssl_param, is_multi_host_flag=None, db_host=None): + if is_multi_host_flag is None: + is_multi_host_flag = bool(db_host and ',' in str(db_host)) + param = existing_ssl_param or '' + if not is_multi_host_flag: + return param + return param + ('&' if param else '?') + 'targetServerType=primary' + conf_dict={} @@ -292,7 +311,6 @@ def init_variables(switch): conf_dict['SQL_CONNECTOR_JAR'] = os.path.join(dir,filename) - conf_dict['db_host']=os.getenv("RANGER_ADMIN_DB_HOST") + ":" + os.getenv("RANGER_ADMIN_DB_PORT") conf_dict['db_name']=os.getenv("RANGER_ADMIN_DB_DBNAME") conf_dict['db_user']=os.getenv("RANGER_ADMIN_DB_USERNAME") conf_dict['db_password']=os.getenv("RANGER_ADMIN_DB_PASSWORD") @@ -812,10 +830,14 @@ def update_properties(): updatePropertyToFilePy(propertyName ,newPropertyValue ,to_file_ranger) elif RANGER_DB_FLAVOR == "POSTGRES": + pg_admin_host = build_pg_host_segment(MYSQL_HOST, RANGER_ADMIN_DB_PORT) + pg_audit_host = build_pg_host_segment(MYSQL_HOST, RANGER_AUDIT_DB_PORT) + pg_query_suffix = build_pg_query_params('', db_host=MYSQL_HOST) + propertyName="ranger.jpa.jdbc.url" - newPropertyValue="jdbc:postgresql://%s:%s/%s" %(MYSQL_HOST, RANGER_ADMIN_DB_PORT, db_name) + newPropertyValue="jdbc:postgresql://%s/%s%s" % (pg_admin_host, db_name, pg_query_suffix) updatePropertyToFilePy(propertyName ,newPropertyValue ,to_file_ranger) - + propertyName="ranger.jpa.jdbc.user" newPropertyValue=db_user updatePropertyToFilePy(propertyName ,newPropertyValue ,to_file_ranger) @@ -823,9 +845,9 @@ def update_properties(): propertyName="ranger.jpa.audit.jdbc.user" newPropertyValue=audit_db_user updatePropertyToFilePy(propertyName ,newPropertyValue ,to_file_ranger) - + propertyName="ranger.jpa.audit.jdbc.url" - newPropertyValue="jdbc:postgresql://%s:%s/%s" %(MYSQL_HOST, RANGER_AUDIT_DB_PORT, audit_db_name) + newPropertyValue="jdbc:postgresql://%s/%s%s" % (pg_audit_host, audit_db_name, pg_query_suffix) updatePropertyToFilePy(propertyName ,newPropertyValue ,to_file_ranger) propertyName="ranger.jpa.jdbc.dialect" diff --git a/security-admin/src/test/python/test_pg_jdbc_util.py b/security-admin/src/test/python/test_pg_jdbc_util.py new file mode 100644 index 00000000000..01c639d48f2 --- /dev/null +++ b/security-admin/src/test/python/test_pg_jdbc_util.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +import unittest + +SCRIPTS_DIR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', 'scripts')) +if SCRIPTS_DIR not in sys.path: + sys.path.insert(0, SCRIPTS_DIR) + +from pg_jdbc_util import ( + DEFAULT_PG_PORT, + TARGET_SERVER_TYPE_PRIMARY, + build_pg_host_segment, + build_pg_jdbc_url, + build_pg_query_params, + is_multi_host, + parse_pg_hosts, +) + + +class TestIsMultiHost(unittest.TestCase): + def test_single_host(self): + self.assertFalse(is_multi_host('localhost')) + self.assertFalse(is_multi_host('host1:5432')) + + def test_multi_host(self): + self.assertTrue(is_multi_host('host1:5432,host2:5432')) + self.assertTrue(is_multi_host('h1,h2')) + + def test_none_and_empty(self): + self.assertFalse(is_multi_host(None)) + self.assertFalse(is_multi_host('')) + + def test_non_string_input(self): + self.assertFalse(is_multi_host(5432)) + self.assertFalse(is_multi_host(0)) + + +class TestParsePgHosts(unittest.TestCase): + def test_host_only(self): + self.assertEqual(parse_pg_hosts('localhost'), [('localhost', DEFAULT_PG_PORT)]) + + def test_host_with_port(self): + self.assertEqual(parse_pg_hosts('db.example.com:5433'), [('db.example.com', 5433)]) + + def test_multi_host_with_ports(self): + self.assertEqual( + parse_pg_hosts('host1:5432,host2:5433'), + [('host1', 5432), ('host2', 5433)], + ) + + def test_multi_host_default_port(self): + self.assertEqual( + parse_pg_hosts('host1,host2'), + [('host1', DEFAULT_PG_PORT), ('host2', DEFAULT_PG_PORT)], + ) + + def test_normalizes_case(self): + self.assertEqual(parse_pg_hosts('HOST1:5432,Host2'), [('host1', 5432), ('host2', DEFAULT_PG_PORT)]) + + def test_empty(self): + self.assertEqual(parse_pg_hosts(''), []) + self.assertEqual(parse_pg_hosts(None), []) + + +class TestBuildPgHostSegment(unittest.TestCase): + def test_single_host_adds_port(self): + self.assertEqual(build_pg_host_segment('localhost'), 'localhost:5432') + self.assertEqual(build_pg_host_segment('localhost', 3306), 'localhost:3306') + + def test_single_host_with_port_preserved(self): + self.assertEqual(build_pg_host_segment('localhost:5433'), 'localhost:5433') + + def test_multi_host_preserved(self): + segment = 'host1:5432,host2:5432' + self.assertEqual(build_pg_host_segment(segment), segment.lower()) + self.assertEqual(build_pg_host_segment(segment, 9999), segment.lower()) + + def test_empty(self): + self.assertEqual(build_pg_host_segment(''), '') + self.assertEqual(build_pg_host_segment(None), '') + + +class TestBuildPgQueryParams(unittest.TestCase): + def test_single_host_no_change(self): + ssl = '?ssl=true&sslmode=verify-full' + self.assertEqual(build_pg_query_params(ssl, is_multi_host_flag=False), ssl) + self.assertEqual(build_pg_query_params('', is_multi_host_flag=False), '') + + def test_multi_host_adds_primary(self): + self.assertEqual( + build_pg_query_params('', is_multi_host_flag=True), + '?%s' % TARGET_SERVER_TYPE_PRIMARY, + ) + + def test_multi_host_with_ssl(self): + ssl = '?ssl=true&sslmode=verify-full' + expected = ssl + '&' + TARGET_SERVER_TYPE_PRIMARY + self.assertEqual(build_pg_query_params(ssl, is_multi_host_flag=True), expected) + + def test_multi_host_detected_from_db_host(self): + self.assertEqual( + build_pg_query_params('?ssl=true', db_host='h1:5432,h2:5432'), + '?ssl=true&%s' % TARGET_SERVER_TYPE_PRIMARY, + ) + + def test_no_duplicate_target_server_type(self): + existing = '?ssl=true&%s' % TARGET_SERVER_TYPE_PRIMARY + self.assertEqual(build_pg_query_params(existing, is_multi_host_flag=True), existing) + + def test_multi_host_with_full_ssl_string(self): + # real db_ssl_param from db_setup.py when db_ssl_certificate_file is set + ssl = '?ssl=true&sslmode=verify-full&sslrootcert=/etc/ssl/certs/ca.pem' + expected = ssl + '&' + TARGET_SERVER_TYPE_PRIMARY + self.assertEqual(build_pg_query_params(ssl, is_multi_host_flag=True), expected) + + def test_multi_host_with_sslfactory_ssl_string(self): + # real db_ssl_param from dba_script.py (NonValidatingFactory) + ssl = '?ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory' + expected = ssl + '&' + TARGET_SERVER_TYPE_PRIMARY + self.assertEqual(build_pg_query_params(ssl, is_multi_host_flag=True), expected) + + def test_param_without_question_mark_multi_host(self): + # non-standard input without '?' — function must prepend '?' correctly + result = build_pg_query_params('ssl=true', is_multi_host_flag=True) + self.assertTrue(result.startswith('?')) + self.assertIn(TARGET_SERVER_TYPE_PRIMARY, result) + + def test_single_host_full_ssl_string_unchanged(self): + # single host — long SSL string must not be modified + ssl = '?ssl=true&sslmode=verify-full&sslrootcert=/etc/ssl/certs/ca.pem' + self.assertEqual(build_pg_query_params(ssl, is_multi_host_flag=False), ssl) + + def test_none_existing_ssl_param(self): + self.assertEqual(build_pg_query_params(None, is_multi_host_flag=False), '') + self.assertEqual( + build_pg_query_params(None, is_multi_host_flag=True), + '?' + TARGET_SERVER_TYPE_PRIMARY, + ) + + +class TestBuildPgJdbcUrl(unittest.TestCase): + def test_basic(self): + url = build_pg_jdbc_url('localhost:5432', 'ranger', '?ssl=true') + self.assertEqual(url, 'jdbc:postgresql://localhost:5432/ranger?ssl=true') + + def test_multi_host_patroni(self): + host = 'h1:5432,h2:5432' + query = build_pg_query_params('', is_multi_host_flag=True) + url = build_pg_jdbc_url(host, 'ranger', query) + self.assertEqual( + url, + 'jdbc:postgresql://h1:5432,h2:5432/ranger?%s' % TARGET_SERVER_TYPE_PRIMARY, + ) + + +if __name__ == '__main__': + unittest.main()