Merge pull request #19974 from qoega/kerberized-hdfs-port-clash

Fixed port clash from test_storage_kerberized_hdfs test.

docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml

@@ -14,7 +14,7 @@ services:
     ports:
       - 1006:1006
       - 50070:50070
-      - 9000:9000
+      - 9010:9010
     depends_on:
         - hdfskerberos
     entrypoint: /etc/bootstrap.sh -d

tests/integration/helpers/cluster.py

@@ -48,9 +48,9 @@ def run_and_check(args, env=None, shell=False):
     res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, shell=shell)
     if res.returncode != 0:
         # check_call(...) from subprocess does not print stderr, so we do it manually
-        print('Stderr:\n{}\n'.format(res.stderr))
-        print('Stdout:\n{}\n'.format(res.stdout))
-        raise Exception('Command {} return non-zero code {}: {}'.format(args, res.returncode, res.stderr))
+        print('Stderr:\n{}\n'.format(res.stderr.decode('utf-8')))
+        print('Stdout:\n{}\n'.format(res.stdout.decode('utf-8')))
+        raise Exception('Command {} return non-zero code {}: {}'.format(args, res.returncode, res.stderr.decode('utf-8')))
 
 
 def subprocess_check_call(args):

tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh

@@ -34,11 +34,11 @@ cat >> /usr/local/hadoop/etc/hadoop/core-site.xml << EOF
    </property>
   <property>
     <name>fs.defaultFS</name>
-    <value>hdfs://kerberizedhdfs1:9000</value>
+    <value>hdfs://kerberizedhdfs1:9010</value>
   </property>
   <property>
     <name>fs.default.name</name>
-    <value>hdfs://kerberizedhdfs1:9000</value>
+    <value>hdfs://kerberizedhdfs1:9010</value>
   </property>
   <!--
   <property>

tests/integration/test_storage_kerberized_hdfs/test.py

@@ -29,12 +29,12 @@ def test_read_table(started_cluster):
     api_read = started_cluster.hdfs_api.read_data("/simple_table_function")
     assert api_read == data
 
-    select_read = node1.query("select * from hdfs('hdfs://kerberizedhdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')")
+    select_read = node1.query("select * from hdfs('hdfs://kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')")
     assert select_read == data
 
 
 def test_read_write_storage(started_cluster):
-    node1.query("create table SimpleHDFSStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9000/simple_storage1', 'TSV')")
+    node1.query("create table SimpleHDFSStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage1', 'TSV')")
     node1.query("insert into SimpleHDFSStorage2 values (1, 'Mark', 72.53)")
 
     api_read = started_cluster.hdfs_api.read_data("/simple_storage1")
@@ -45,7 +45,7 @@ def test_read_write_storage(started_cluster):
 
 
 def test_write_storage_not_expired(started_cluster):
-    node1.query("create table SimpleHDFSStorageNotExpired (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9000/simple_storage_not_expired', 'TSV')")
+    node1.query("create table SimpleHDFSStorageNotExpired (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage_not_expired', 'TSV')")
 
     time.sleep(45)   # wait for ticket expiration
     node1.query("insert into SimpleHDFSStorageNotExpired values (1, 'Mark', 72.53)")
@@ -58,15 +58,15 @@ def test_write_storage_not_expired(started_cluster):
 
 
 def test_two_users(started_cluster):
-    node1.query("create table HDFSStorOne (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9000/storage_user_one', 'TSV')")
+    node1.query("create table HDFSStorOne (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/storage_user_one', 'TSV')")
     node1.query("insert into HDFSStorOne values (1, 'Real', 86.00)")
 
-    node1.query("create table HDFSStorTwo (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9000/user/specuser/storage_user_two', 'TSV')")
+    node1.query("create table HDFSStorTwo (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV')")
     node1.query("insert into HDFSStorTwo values (1, 'Ideal', 74.00)")
 
-    select_read_1 = node1.query("select * from hdfs('hdfs://kerberizedhdfs1:9000/user/specuser/storage_user_two', 'TSV', 'id UInt64, text String, number Float64')")
+    select_read_1 = node1.query("select * from hdfs('hdfs://kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV', 'id UInt64, text String, number Float64')")
 
-    select_read_2 = node1.query("select * from hdfs('hdfs://suser@kerberizedhdfs1:9000/storage_user_one', 'TSV', 'id UInt64, text String, number Float64')")
+    select_read_2 = node1.query("select * from hdfs('hdfs://suser@kerberizedhdfs1:9010/storage_user_one', 'TSV', 'id UInt64, text String, number Float64')")
 
 def test_read_table_expired(started_cluster):
     data = "1\tSerialize\t555.222\n2\tData\t777.333\n"
@@ -76,7 +76,7 @@ def test_read_table_expired(started_cluster):
     time.sleep(45)
 
     try:
-        select_read = node1.query("select * from hdfs('hdfs://reloginuser&kerberizedhdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')")
+        select_read = node1.query("select * from hdfs('hdfs://reloginuser&kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')")
         assert False, "Exception have to be thrown"
     except Exception as ex:
         assert "DB::Exception: kinit failure:" in str(ex)
@@ -85,7 +85,7 @@ def test_read_table_expired(started_cluster):
 
 
 def test_prohibited(started_cluster):
-    node1.query("create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9000/storage_user_two_prohibited', 'TSV')")
+    node1.query("create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')")
     try:
         node1.query("insert into HDFSStorTwoProhibited values (1, 'SomeOne', 74.00)")
         assert False, "Exception have to be thrown"
@@ -94,7 +94,7 @@ def test_prohibited(started_cluster):
 
 
 def test_cache_path(started_cluster):
-    node1.query("create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9000/storage_dedicated_cache_path', 'TSV')")
+    node1.query("create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')")
     try:
         node1.query("insert into HDFSStorCachePath values (1, 'FatMark', 92.53)")
         assert False, "Exception have to be thrown"