+ Redirect

dbms/src/Common/ErrorCodes.cpp

@@ -452,6 +452,10 @@ namespace ErrorCodes
     extern const int INVALID_WITH_FILL_EXPRESSION = 475;
     extern const int WITH_TIES_WITHOUT_ORDER_BY = 476;
     extern const int INVALID_USAGE_OF_INPUT = 477;
+    extern const int TOO_MANY_REDIRECTS = 478;
+ 
+    extern const int KEEPER_EXCEPTION = 999;
+    extern const int POCO_EXCEPTION = 1000;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Core/Settings.h

@@ -169,6 +169,8 @@ struct Settings : public SettingsCollection<Settings>
     \
     M(SettingBool, add_http_cors_header, false, "Write add http CORS header.") \
     \
+    M(SettingUInt64, max_http_get_redirects, 0, "Max number of http GET redirects hops allowed.") \
+    \
     M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).") \
     M(SettingBool, input_format_with_names_use_header, false, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.") \
     M(SettingBool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).") \

dbms/src/IO/HTTPCommon.cpp

@@ -40,6 +40,7 @@ namespace ErrorCodes
     extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS;
     extern const int FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME;
     extern const int UNSUPPORTED_URI_SCHEME;
+    extern const int TOO_MANY_REDIRECTS;
 }
 
 
@@ -223,8 +224,15 @@ std::istream * receiveResponse(
     auto istr = &session.receiveResponse(response);
     auto status = response.getStatus();
 
-    if (status != Poco::Net::HTTPResponse::HTTP_OK)
-    {
+    if (
+        ( request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) &&  //  we only accepts redirects on GET requests.
+        (status == Poco::Net::HTTPResponse::HTTP_MOVED_PERMANENTLY  || // 301
+            status == Poco::Net::HTTPResponse::HTTP_FOUND || // 302
+            status == Poco::Net::HTTPResponse::HTTP_SEE_OTHER  || // 303
+            status == Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT) // 307
+        )  {
+        throw Poco::URIRedirection(response.get("Location"));
+    } else if (status != Poco::Net::HTTPResponse::HTTP_OK)    {
         std::stringstream error_message;
         error_message << "Received error from remote server " << request.getURI() << ". HTTP status code: " << status << " "
                       << response.getReason() << ", body: " << istr->rdbuf();

dbms/src/IO/HTTPCommon.h

@@ -9,7 +9,7 @@
 #include <Poco/Net/HTTPResponse.h>
 #include <Poco/URI.h>
 #include <Common/PoolBase.h>
-
+#include <Poco/URIStreamFactory.h>
 
 #include <IO/ConnectionTimeouts.h>
 

dbms/src/IO/ReadWriteBufferFromHTTP.cpp

 #include <IO/ReadWriteBufferFromHTTP.h>
+#include <Common/Exception.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+	extern const int TOO_MANY_REDIRECTS; 
+}
+
+
+std::unique_ptr<DB::ReadWriteBufferFromHTTP> makeReadWriteBufferFromHTTP(const Poco::URI & uri,
+	const std::string & method,
+	std::function<void(std::ostream &)> callback,
+	const DB::ConnectionTimeouts & timeouts,
+	const DB::SettingUInt64 max_redirects)
+	{
+		auto actual_uri =uri;
+		UInt64 redirects = 0;
+		
+		do 
+		{
+			try 
+			{
+				return std::make_unique<DB::ReadWriteBufferFromHTTP>(actual_uri, method, callback, timeouts);
+			} 
+			catch (Poco::URIRedirection & exc) {
+				redirects++;
+				actual_uri = exc.uri();
+			}
+		} while(max_redirects>redirects);
+		
+		// too many redirects....
+		std::stringstream error_message;
+		error_message << "Too many redirects while trying to access " << uri.toString() ;
+		
+		throw Exception(error_message.str(), ErrorCodes::TOO_MANY_REDIRECTS);
+	}
+}
\ No newline at end of file

dbms/src/IO/ReadWriteBufferFromHTTP.h

@@ -16,6 +16,7 @@
 #include <Common/DNSResolver.h>
 #include <Common/config.h>
 #include <common/logger_useful.h>
+#include <Poco/URIStreamFactory.h>
 
 
 #define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
@@ -139,5 +140,10 @@ public:
     }
 };
 
+std::unique_ptr<ReadWriteBufferFromHTTP> makeReadWriteBufferFromHTTP(const Poco::URI & uri,
+    const std::string & method,
+    std::function<void(std::ostream &)> callback,
+    const ConnectionTimeouts & timeouts,
+    const SettingUInt64 max_redirects);
 
 }

dbms/src/Storages/StorageURL.cpp

@@ -54,8 +54,7 @@ namespace
             const ConnectionTimeouts & timeouts)
             : name(name_)
         {
-            read_buf = std::make_unique<ReadWriteBufferFromHTTP>(uri, method, callback, timeouts);
-
+            read_buf = makeReadWriteBufferFromHTTP(uri, method, callback, timeouts,context.getSettingsRef().max_http_get_redirects);
             reader = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size);
         }