diff --git a/core/src/main/java/io/questdb/cutlass/http/HttpConnectionContext.java b/core/src/main/java/io/questdb/cutlass/http/HttpConnectionContext.java index 4895786d0abad6702b5eb3cbb3e741c82cd1efd1..2163773e09f8ce4a94ca73524adf1a9fab27cacb 100644 --- a/core/src/main/java/io/questdb/cutlass/http/HttpConnectionContext.java +++ b/core/src/main/java/io/questdb/cutlass/http/HttpConnectionContext.java @@ -406,7 +406,7 @@ public class HttpConnectionContext implements IOContext, Locality, Mutable { private boolean rejectRequest(CharSequence userMessage) throws PeerDisconnectedException, PeerIsSlowToReadException { clear(); LOG.error().$(userMessage).$(); - simpleResponse().sendStatus(400, userMessage); + simpleResponse().sendStatus(404, userMessage); dispatcher.registerChannel(this, IOOperation.READ); return false; } diff --git a/core/src/main/java/io/questdb/cutlass/http/HttpResponseSink.java b/core/src/main/java/io/questdb/cutlass/http/HttpResponseSink.java index 47b007ebc36b23cefc642f75f69327afc0eec9d3..ac7963a61c3a79f7ee6c6e9859da1232a33daaaa 100644 --- a/core/src/main/java/io/questdb/cutlass/http/HttpResponseSink.java +++ b/core/src/main/java/io/questdb/cutlass/http/HttpResponseSink.java @@ -477,7 +477,7 @@ public class HttpResponseSink implements Closeable, Mutable { public class SimpleResponseImpl { public void sendStatus(int code, CharSequence message) throws PeerDisconnectedException, PeerIsSlowToReadException { - final String std = headerImpl.status(httpVersion, code, "text/html; charset=utf-8", -1L); + final String std = headerImpl.status(httpVersion, code, "text/plain; charset=utf-8", -1L); sink.put(message == null ? std : message).put(Misc.EOL); prepareHeaderSink(); resumeSend(CHUNK_HEAD); diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessor.java b/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessor.java index 2b59539476539a1a6d7ba579fc11724dd5a9b4d0..b28eef7415fd6589005a81a61ee0d737ac992a61 100644 --- a/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessor.java +++ b/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessor.java @@ -24,10 +24,6 @@ package io.questdb.cutlass.http.processors; -import java.io.Closeable; - -import org.jetbrains.annotations.Nullable; - import io.questdb.MessageBus; import io.questdb.Telemetry; import io.questdb.cairo.CairoEngine; @@ -37,40 +33,30 @@ import io.questdb.cairo.sql.InsertMethod; import io.questdb.cairo.sql.InsertStatement; import io.questdb.cairo.sql.ReaderOutOfDateException; import io.questdb.cairo.sql.RecordCursorFactory; -import io.questdb.cutlass.http.HttpChunkedResponseSocket; -import io.questdb.cutlass.http.HttpConnectionContext; -import io.questdb.cutlass.http.HttpRequestHeader; -import io.questdb.cutlass.http.HttpRequestProcessor; -import io.questdb.cutlass.http.LocalValue; +import io.questdb.cutlass.http.*; import io.questdb.cutlass.text.Utf8Exception; -import io.questdb.griffin.CompiledQuery; -import io.questdb.griffin.FunctionFactoryCache; -import io.questdb.griffin.SqlCompiler; -import io.questdb.griffin.SqlException; -import io.questdb.griffin.SqlExecutionContextImpl; +import io.questdb.griffin.*; import io.questdb.log.Log; import io.questdb.log.LogFactory; import io.questdb.network.NoSpaceLeftInResponseBufferException; import io.questdb.network.PeerDisconnectedException; import io.questdb.network.PeerIsSlowToReadException; import io.questdb.network.ServerDisconnectException; -import io.questdb.std.Chars; -import io.questdb.std.Misc; -import io.questdb.std.NanosecondClock; -import io.questdb.std.Numbers; -import io.questdb.std.NumericException; -import io.questdb.std.ObjList; +import io.questdb.std.*; import io.questdb.std.str.DirectByteCharSequence; import io.questdb.std.str.Path; +import org.jetbrains.annotations.Nullable; + +import java.io.Closeable; public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { private static final LocalValue LV = new LocalValue<>(); private static final Log LOG = LogFactory.getLog(JsonQueryProcessor.class); + protected final ObjList queryExecutors = new ObjList<>(); private final SqlCompiler compiler; private final JsonQueryProcessorConfiguration configuration; private final SqlExecutionContextImpl sqlExecutionContext; private final Path path = new Path(); - protected final ObjList queryExecutors = new ObjList<>(); private final NanosecondClock nanosecondClock; public JsonQueryProcessor( @@ -79,7 +65,7 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { @Nullable MessageBus messageBus, int workerCount ) { - this(configuration, engine, messageBus, workerCount, null); + this(configuration, engine, messageBus, workerCount, (FunctionFactoryCache) null); } public JsonQueryProcessor( @@ -89,7 +75,7 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { int workerCount, @Nullable FunctionFactoryCache functionFactoryCache ) { - this(configuration, engine, messageBus, workerCount, null, new SqlCompiler(engine, messageBus, functionFactoryCache)); + this(configuration, engine, messageBus, workerCount, new SqlCompiler(engine, messageBus, functionFactoryCache)); } public JsonQueryProcessor( @@ -97,7 +83,6 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { CairoEngine engine, @Nullable MessageBus messageBus, int workerCount, - @Nullable FunctionFactoryCache functionFactoryCache, SqlCompiler sqlCompiler ) { this.configuration = configuration; @@ -120,110 +105,6 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { this.nanosecondClock = engine.getConfiguration().getNanosecondClock(); } - private static void doResumeSend( - JsonQueryProcessorState state, - HttpConnectionContext context - ) throws PeerDisconnectedException, PeerIsSlowToReadException { - if (state.noCursor()) { - return; - } - - LOG.debug().$("resume [fd=").$(context.getFd()).$(']').$(); - - final HttpChunkedResponseSocket socket = context.getChunkedResponseSocket(); - while (true) { - try { - state.resume(socket); - break; - } catch (NoSpaceLeftInResponseBufferException ignored) { - if (socket.resetToBookmark()) { - socket.sendChunk(); - } else { - // what we have here is out unit of data, column value or query - // is larger that response content buffer - // all we can do in this scenario is to log appropriately - // and disconnect socket - state.logBufferTooSmall(); - throw PeerDisconnectedException.INSTANCE; - } - } - } - // reached the end naturally? - readyForNextRequest(context); - } - - private static void cannotCopyRemote( - JsonQueryProcessorState state, - CompiledQuery cc, - CharSequence keepAliveHeader - ) throws SqlException { - throw SqlException.$(0, "copy from STDIN is not supported over REST"); - } - - protected static void header( - HttpChunkedResponseSocket socket, - int status, - CharSequence keepAliveHeader - ) throws PeerDisconnectedException, PeerIsSlowToReadException { - socket.status(status, "application/json; charset=utf-8"); - socket.headers().setKeepAlive(keepAliveHeader); - socket.sendHeader(); - } - - private static void readyForNextRequest(HttpConnectionContext context) { - LOG.info().$("all sent [fd=").$(context.getFd()).$(", lastRequestBytesSent=").$(context.getLastRequestBytesSent()).$(", nCompletedRequests=").$(context.getNCompletedRequests() + 1) - .$(", totalBytesSent=").$(context.getTotalBytesSent()).$(']').$(); - } - - protected static void sendConfirmation( - JsonQueryProcessorState state, - CompiledQuery cq, - CharSequence keepAliveHeader - ) throws PeerDisconnectedException, PeerIsSlowToReadException { - final HttpConnectionContext context = state.getHttpConnectionContext(); - final HttpChunkedResponseSocket socket = context.getChunkedResponseSocket(); - header(socket, 200, keepAliveHeader); - socket.put('{').putQuoted("ddl").put(':').putQuoted("OK").put('}'); - socket.sendChunk(); - socket.done(); - readyForNextRequest(context); - } - - static void sendException( - HttpChunkedResponseSocket socket, - int position, - CharSequence message, - int status, - CharSequence query, - CharSequence keepAliveHeader - ) throws PeerDisconnectedException, PeerIsSlowToReadException { - header(socket, status, keepAliveHeader); - socket.put('{'). - putQuoted("query").put(':').encodeUtf8AndQuote(query == null ? "" : query).put(','). - putQuoted("error").put(':').encodeUtf8AndQuote(message).put(','). - putQuoted("position").put(':').put(position); - socket.put('}'); - socket.sendChunk(); - socket.done(); - } - - private static void syntaxError( - HttpChunkedResponseSocket socket, - SqlException sqlException, - JsonQueryProcessorState state, - CharSequence keepAliveHeader - ) throws PeerDisconnectedException, PeerIsSlowToReadException { - state.logSyntaxError(sqlException); - sendException( - socket, - sqlException.getPosition(), - sqlException.getFlyweightMessage(), - 400, - state.getQuery(), - keepAliveHeader - ); - } - @Override public void close() { Misc.free(compiler); @@ -314,33 +195,99 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { } } - private void executeCachedSelect( + private static void doResumeSend( JsonQueryProcessorState state, - RecordCursorFactory factory, + HttpConnectionContext context + ) throws PeerDisconnectedException, PeerIsSlowToReadException { + if (state.noCursor()) { + return; + } + + LOG.debug().$("resume [fd=").$(context.getFd()).$(']').$(); + + final HttpChunkedResponseSocket socket = context.getChunkedResponseSocket(); + while (true) { + try { + state.resume(socket); + break; + } catch (NoSpaceLeftInResponseBufferException ignored) { + if (socket.resetToBookmark()) { + socket.sendChunk(); + } else { + // what we have here is out unit of data, column value or query + // is larger that response content buffer + // all we can do in this scenario is to log appropriately + // and disconnect socket + state.logBufferTooSmall(); + throw PeerDisconnectedException.INSTANCE; + } + } + } + // reached the end naturally? + readyForNextRequest(context); + } + + private static void cannotCopyRemote( + JsonQueryProcessorState state, + CompiledQuery cc, + CharSequence keepAliveHeader + ) throws SqlException { + throw SqlException.$(0, "copy from STDIN is not supported over REST"); + } + + protected static void header( + HttpChunkedResponseSocket socket, CharSequence keepAliveHeader ) throws PeerDisconnectedException, PeerIsSlowToReadException { - state.setCompilerNanos(0); - state.logExecuteCached(); - executeSelect(state, factory, keepAliveHeader); + socket.status(200, "application/json; charset=utf-8"); + socket.headers().setKeepAlive(keepAliveHeader); + socket.sendHeader(); } - private void executeSelect( + private static void readyForNextRequest(HttpConnectionContext context) { + LOG.info().$("all sent [fd=").$(context.getFd()).$(", lastRequestBytesSent=").$(context.getLastRequestBytesSent()).$(", nCompletedRequests=").$(context.getNCompletedRequests() + 1) + .$(", totalBytesSent=").$(context.getTotalBytesSent()).$(']').$(); + } + + protected static void sendConfirmation( JsonQueryProcessorState state, - RecordCursorFactory factory, + CompiledQuery cq, CharSequence keepAliveHeader ) throws PeerDisconnectedException, PeerIsSlowToReadException { final HttpConnectionContext context = state.getHttpConnectionContext(); - try { - if (state.of(factory, sqlExecutionContext)) { - header(context.getChunkedResponseSocket(), 200, keepAliveHeader); - doResumeSend(state, context); - } else { - readyForNextRequest(context); - } - } catch (CairoException ex) { - state.setQueryCacheable(ex.isCacheable()); - throw ex; - } + final HttpChunkedResponseSocket socket = context.getChunkedResponseSocket(); + header(socket, keepAliveHeader); + socket.put('{').putQuoted("ddl").put(':').putQuoted("OK").put('}'); + socket.sendChunk(); + socket.done(); + readyForNextRequest(context); + } + + static void sendException( + HttpChunkedResponseSocket socket, + int position, + CharSequence message, + CharSequence query, + CharSequence keepAliveHeader + ) throws PeerDisconnectedException, PeerIsSlowToReadException { + header(socket, keepAliveHeader); + JsonQueryProcessorState.prepareExceptionJson(socket, position, message, query); + } + + private static void syntaxError( + HttpChunkedResponseSocket socket, + SqlException sqlException, + JsonQueryProcessorState state, + CharSequence keepAliveHeader + ) throws PeerDisconnectedException, PeerIsSlowToReadException { + state.logSyntaxError(sqlException); + sendException( + socket, + sqlException.getPosition(), + sqlException.getFlyweightMessage(), + state.getQuery(), + keepAliveHeader + ); } private void compileQuery(JsonQueryProcessorState state) throws SqlException, PeerDisconnectedException, PeerIsSlowToReadException { @@ -355,6 +302,16 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { ); } + private void executeCachedSelect( + JsonQueryProcessorState state, + RecordCursorFactory factory, + CharSequence keepAliveHeader + ) throws PeerDisconnectedException, PeerIsSlowToReadException { + state.setCompilerNanos(0); + state.logExecuteCached(); + executeSelect(state, factory, keepAliveHeader); + } + private void executeInsert( JsonQueryProcessorState state, CompiledQuery cc, @@ -381,6 +338,25 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { keepAliveHeader); } + private void executeSelect( + JsonQueryProcessorState state, + RecordCursorFactory factory, + CharSequence keepAliveHeader + ) throws PeerDisconnectedException, PeerIsSlowToReadException { + final HttpConnectionContext context = state.getHttpConnectionContext(); + try { + if (state.of(factory, sqlExecutionContext)) { + header(context.getChunkedResponseSocket(), keepAliveHeader); + doResumeSend(state, context); + } else { + readyForNextRequest(context); + } + } catch (CairoException ex) { + state.setQueryCacheable(ex.isCacheable()); + throw ex; + } + } + private void internalError( HttpChunkedResponseSocket socket, CharSequence message, @@ -392,7 +368,7 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { } else { state.error().$("internal error [q=`").utf8(state.getQuery()).$("`, ex=").$(e).$(']').$(); } - sendException(socket, 0, message, 500, state.getQuery(), configuration.getKeepAliveHeader()); + sendException(socket, 0, message, state.getQuery(), configuration.getKeepAliveHeader()); } private boolean parseUrl( @@ -404,7 +380,7 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { final DirectByteCharSequence query = header.getUrlParam("query"); if (query == null || query.length() == 0) { state.info().$("Empty query header received. Sending empty reply.").$(); - sendException(state.getHttpConnectionContext().getChunkedResponseSocket(), 0, "No query text", 400, query, keepAliveHeader); + sendException(state.getHttpConnectionContext().getChunkedResponseSocket(), 0, "No query text", query, keepAliveHeader); return false; } @@ -444,7 +420,7 @@ public class JsonQueryProcessor implements HttpRequestProcessor, Closeable { state.configure(header, query, skip, stop); } catch (Utf8Exception e) { state.info().$("Bad UTF8 encoding").$(); - sendException(state.getHttpConnectionContext().getChunkedResponseSocket(), 0, "Bad UTF8 encoding in query text", 400, query, keepAliveHeader); + sendException(state.getHttpConnectionContext().getChunkedResponseSocket(), 0, "Bad UTF8 encoding in query text", query, keepAliveHeader); return false; } return true; diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessorState.java b/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessorState.java index 19dffee06df8cda98625add8aceb680b8480c508..ffddb35229c9bb2a5bf0437090845da64bec7643 100644 --- a/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessorState.java +++ b/core/src/main/java/io/questdb/cutlass/http/processors/JsonQueryProcessorState.java @@ -200,10 +200,6 @@ public class JsonQueryProcessorState implements Mutable, Closeable { return query; } - void setQueryCacheable(boolean queryCacheable) { - this.queryCacheable = queryCacheable; - } - public Rnd getRnd() { return rnd; } @@ -231,6 +227,10 @@ public class JsonQueryProcessorState implements Mutable, Closeable { $(']').$(); } + public void logSyntaxError(SqlException e) { + info().$("syntax-error [q=`").utf8(query).$("`, at=").$(e.getPosition()).$(", message=`").utf8(e.getFlyweightMessage()).$('`').$(']').$(); + } + public void logTimings() { info().$("timings "). $("[compiler: ").$(compilerNanos). @@ -240,10 +240,6 @@ public class JsonQueryProcessorState implements Mutable, Closeable { $("`]").$(); } - public void logSyntaxError(SqlException e) { - info().$("syntax-error [q=`").utf8(query).$("`, at=").$(e.getPosition()).$(", message=`").utf8(e.getFlyweightMessage()).$('`').$(']').$(); - } - public void setCompilerNanos(long compilerNanos) { this.compilerNanos = compilerNanos; } @@ -252,6 +248,16 @@ public class JsonQueryProcessorState implements Mutable, Closeable { this.executeStartNanos = nanosecondClock.getTicks(); } + static void prepareExceptionJson(HttpChunkedResponseSocket socket, int position, CharSequence message, CharSequence query) throws PeerDisconnectedException, PeerIsSlowToReadException { + socket.put('{'). + putQuoted("query").put(':').encodeUtf8AndQuote(query == null ? "" : query).put(','). + putQuoted("error").put(':').encodeUtf8AndQuote(message).put(','). + putQuoted("position").put(':').put(position); + socket.put('}'); + socket.sendChunk(); + socket.done(); + } + private static void putStringOrNull(CharSink r, CharSequence str) { if (str == null) { r.put("null"); @@ -336,12 +342,16 @@ public class JsonQueryProcessorState implements Mutable, Closeable { socket.put('"').putISODate(t).put('"'); } + private static void putCursorValue(HttpChunkedResponseSocket socket, Record rec, int col) { + putStringOrNull(socket, null); + } + private boolean addColumnToOutput(RecordMetadata metadata, CharSequence columnNames, int start, int hi) throws PeerDisconnectedException, PeerIsSlowToReadException { if (start == hi) { info().$("empty column in list '").$(columnNames).$('\'').$(); HttpChunkedResponseSocket socket = getHttpConnectionContext().getChunkedResponseSocket(); - JsonQueryProcessor.header(socket, 400, ""); + JsonQueryProcessor.header(socket, ""); socket.put('{'). putQuoted("query").put(':').encodeUtf8AndQuote(query).put(','). putQuoted("error").put(':').putQuoted("empty column in list"); @@ -355,7 +365,7 @@ public class JsonQueryProcessorState implements Mutable, Closeable { if (columnIndex == RecordMetadata.COLUMN_NOT_FOUND) { info().$("invalid column in list: '").$(columnNames, start, hi).$('\'').$(); HttpChunkedResponseSocket socket = getHttpConnectionContext().getChunkedResponseSocket(); - JsonQueryProcessor.header(socket, 400, ""); + JsonQueryProcessor.header(socket, ""); socket.put('{'). putQuoted("query").put(':').encodeUtf8AndQuote(query).put(','). putQuoted("error").put(':').put('\'').put("invalid column in list: ").put(columnNames, start, hi).put('\''); @@ -528,7 +538,7 @@ public class JsonQueryProcessorState implements Mutable, Closeable { } catch (Utf8Exception e) { info().$("utf8 error when decoding column list '").$(columnNames).$('\'').$(); HttpChunkedResponseSocket socket = getHttpConnectionContext().getChunkedResponseSocket(); - JsonQueryProcessor.header(socket, 400, ""); + JsonQueryProcessor.header(socket, ""); socket.put('{'). putQuoted("error").put(':').putQuoted("utf8 error in column list"); socket.put('}'); @@ -713,10 +723,6 @@ public class JsonQueryProcessorState implements Mutable, Closeable { putStrValue(socket, rec, columnSkewList.getQuick(col)); } - private static void putCursorValue(HttpChunkedResponseSocket socket, Record rec, int col) { - putStringOrNull(socket, null); - } - private void putSkewedSymValue(HttpChunkedResponseSocket socket, Record rec, int col) { putSymValue(socket, rec, columnSkewList.getQuick(col)); } @@ -729,6 +735,10 @@ public class JsonQueryProcessorState implements Mutable, Closeable { resumeActions.getQuick(queryState).onResume(socket, columnCount); } + void setQueryCacheable(boolean queryCacheable) { + this.queryCacheable = queryCacheable; + } + @FunctionalInterface interface StateResumeAction { void onResume( diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/TableStatusCheckProcessor.java b/core/src/main/java/io/questdb/cutlass/http/processors/TableStatusCheckProcessor.java index f5200db8d94e177d84b174890adf2ea523e8183e..7276f860892a177a7bc1498612a4a582878e732b 100644 --- a/core/src/main/java/io/questdb/cutlass/http/processors/TableStatusCheckProcessor.java +++ b/core/src/main/java/io/questdb/cutlass/http/processors/TableStatusCheckProcessor.java @@ -68,7 +68,7 @@ public class TableStatusCheckProcessor implements HttpRequestProcessor, Closeabl public void onRequestComplete(HttpConnectionContext context) throws PeerDisconnectedException, PeerIsSlowToReadException { CharSequence tableName = context.getRequestHeader().getUrlParam("j"); if (tableName == null) { - context.simpleResponse().sendStatus(400, "table name missing"); + context.simpleResponse().sendStatus(200, "table name missing"); } else { int check = cairoEngine.getStatus(context.getCairoSecurityContext(), path, tableName); if (Chars.equalsNc("json", context.getRequestHeader().getUrlParam("f"))) { diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/TextImportProcessor.java b/core/src/main/java/io/questdb/cutlass/http/processors/TextImportProcessor.java index 61d263c193175bc5e8959c31c88144026f641ebe..6d3fee6ba9d92e2e4a07bf025aa0b1200938d64e 100644 --- a/core/src/main/java/io/questdb/cutlass/http/processors/TextImportProcessor.java +++ b/core/src/main/java/io/questdb/cutlass/http/processors/TextImportProcessor.java @@ -62,12 +62,6 @@ public class TextImportProcessor implements HttpRequestProcessor, HttpMultipartC // processor. For different threads to lookup the same value from local value map the key, // which is LV, has to be the same between processor instances private static final LocalValue LV = new LocalValue<>(); - - static { - atomicityParamMap.put("skipRow", Atomicity.SKIP_ROW); - atomicityParamMap.put("abort", Atomicity.SKIP_ALL); - } - private final CairoEngine engine; private HttpConnectionContext transientContext; private TextImportProcessorState transientState; @@ -76,6 +70,129 @@ public class TextImportProcessor implements HttpRequestProcessor, HttpMultipartC this.engine = cairoEngine; } + @Override + public void close() { + } + + @Override + public void onChunk(long lo, long hi) + throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { + if (hi > lo) { + try { + transientState.textLoader.parse(lo, hi, transientContext.getCairoSecurityContext()); + if (transientState.messagePart == MESSAGE_DATA && !transientState.analysed) { + transientState.analysed = true; + transientState.textLoader.setState(TextLoader.LOAD_DATA); + } + } catch (TextException | CairoException | CairoError e) { + handleTextException(e); + } + } + } + + @Override + public void onPartBegin(HttpRequestHeader partHeader) throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { + final CharSequence contentDisposition = partHeader.getContentDispositionName(); + LOG.debug().$("part begin [name=").$(contentDisposition).$(']').$(); + if (Chars.equalsNc("data", contentDisposition)) { + + final HttpRequestHeader rh = transientContext.getRequestHeader(); + CharSequence name = rh.getUrlParam("name"); + if (name == null) { + name = partHeader.getContentDispositionFilename(); + } + + if (name == null) { + sendError("no file name given"); + throw ServerDisconnectException.INSTANCE; + } + + CharSequence partitionedBy = rh.getUrlParam("partitionBy"); + if (partitionedBy == null) { + partitionedBy = "NONE"; + } + int partitionBy = PartitionBy.fromString(partitionedBy); + if (partitionBy == -1) { + sendError("invalid partitionBy"); + throw ServerDisconnectException.INSTANCE; + } + + CharSequence timestampIndexCol = rh.getUrlParam("timestamp"); + if (partitionBy != PartitionBy.NONE && timestampIndexCol == null) { + sendError("when specifying partitionBy you must also specify timestamp"); + throw ServerDisconnectException.INSTANCE; + } + + transientState.analysed = false; + transientState.textLoader.configureDestination( + name, + Chars.equalsNc("true", rh.getUrlParam("overwrite")), + Chars.equalsNc("true", rh.getUrlParam("durable")), + getAtomicity(rh.getUrlParam("atomicity")), + partitionBy, + timestampIndexCol + ); + transientState.textLoader.setForceHeaders(Chars.equalsNc("true", rh.getUrlParam("forceHeader"))); + transientState.textLoader.setSkipRowsWithExtraValues(Chars.equalsNc("true", rh.getUrlParam("skipLev"))); + transientState.textLoader.setState(TextLoader.ANALYZE_STRUCTURE); + + transientState.forceHeader = Chars.equalsNc("true", rh.getUrlParam("forceHeader")); + transientState.messagePart = MESSAGE_DATA; + } else if (Chars.equalsNc("schema", contentDisposition)) { + transientState.textLoader.setState(TextLoader.LOAD_JSON_METADATA); + transientState.messagePart = MESSAGE_SCHEMA; + } else { + if (partHeader.getContentDisposition() == null) { + sendError("'Content-Disposition' multipart header missing'"); + } else { + sendError("invalid value in 'Content-Disposition' multipart header"); + } + throw ServerDisconnectException.INSTANCE; + } + } + + // This processor implements HttpMultipartContentListener, methods of which + // have neither context nor dispatcher. During "chunk" processing we may need + // to send something back to client, or disconnect them. To do that we need + // these transient references. resumeRecv() will set them and they will remain + // valid during multipart events. + + @Override + public void onPartEnd() throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { + try { + LOG.debug().$("part end").$(); + transientState.textLoader.wrapUp(); + if (transientState.messagePart == MESSAGE_DATA) { + sendResponse(transientContext); + } + } catch (TextException | CairoException | CairoError e) { + handleTextException(e); + } + } + + @Override + public void onRequestComplete(HttpConnectionContext context) { + transientState.clear(); + } + + @Override + public void resumeRecv(HttpConnectionContext context) { + this.transientContext = context; + this.transientState = LV.get(context); + if (this.transientState == null) { + LOG.debug().$("new text state").$(); + LV.set(context, this.transientState = new TextImportProcessorState(engine)); + transientState.json = isJson(context); + } + } + + @Override + public void resumeSend( + HttpConnectionContext context + ) throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { + doResumeSend(LV.get(context), context.getChunkedResponseSocket()); + } + private static void resumeJson(TextImportProcessorState state, HttpChunkedResponseSocket socket) throws PeerDisconnectedException, PeerIsSlowToReadException { final TextLoader textLoader = state.textLoader; final RecordMetadata metadata = textLoader.getMetadata(); @@ -141,12 +258,6 @@ public class TextImportProcessor implements HttpRequestProcessor, HttpMultipartC return b; } - // This processor implements HttpMultipartContentListener, methods of which - // have neither context nor dispatcher. During "chunk" processing we may need - // to send something back to client, or disconnect them. To do that we need - // these transient references. resumeRecv() will set them and they will remain - // valid during multipart events. - private static void pad(CharSink b, int w, long value) { int len = (int) Math.log10(value); if (len < 0) { @@ -248,122 +359,6 @@ public class TextImportProcessor implements HttpRequestProcessor, HttpMultipartC return atomicity == -1 ? Atomicity.SKIP_COL : atomicity; } - @Override - public void close() { - } - - @Override - public void onChunk(long lo, long hi) - throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { - if (hi > lo) { - try { - transientState.textLoader.parse(lo, hi, transientContext.getCairoSecurityContext()); - if (transientState.messagePart == MESSAGE_DATA && !transientState.analysed) { - transientState.analysed = true; - transientState.textLoader.setState(TextLoader.LOAD_DATA); - } - } catch (TextException | CairoException | CairoError e) { - handleTextException(e); - } - } - } - - @Override - public void onPartBegin(HttpRequestHeader partHeader) throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { - final CharSequence contentDisposition = partHeader.getContentDispositionName(); - LOG.debug().$("part begin [name=").$(contentDisposition).$(']').$(); - if (Chars.equalsNc("data", contentDisposition)) { - - final HttpRequestHeader rh = transientContext.getRequestHeader(); - CharSequence name = rh.getUrlParam("name"); - if (name == null) { - name = partHeader.getContentDispositionFilename(); - } - if (name == null) { - transientContext.simpleResponse().sendStatus(400, "no file name given"); - throw ServerDisconnectException.INSTANCE; - } - - CharSequence partitionedBy = rh.getUrlParam("partitionBy"); - if (partitionedBy == null) { - partitionedBy = "NONE"; - } - int partitionBy = PartitionBy.fromString(partitionedBy); - if (partitionBy == -1) { - transientContext.simpleResponse().sendStatus(400, "invalid partitionBy"); - throw ServerDisconnectException.INSTANCE; - } - - CharSequence timestampIndexCol = rh.getUrlParam("timestamp"); - if (partitionBy != PartitionBy.NONE && timestampIndexCol == null) { - transientContext.simpleResponse().sendStatus(400, "when specifying partitionBy you must also specify timestamp"); - throw ServerDisconnectException.INSTANCE; - } - - transientState.analysed = false; - transientState.textLoader.configureDestination( - name, - Chars.equalsNc("true", rh.getUrlParam("overwrite")), - Chars.equalsNc("true", rh.getUrlParam("durable")), - getAtomicity(rh.getUrlParam("atomicity")), - partitionBy, - timestampIndexCol - ); - transientState.textLoader.setForceHeaders(Chars.equalsNc("true", rh.getUrlParam("forceHeader"))); - transientState.textLoader.setSkipRowsWithExtraValues(Chars.equalsNc("true", rh.getUrlParam("skipLev"))); - transientState.textLoader.setState(TextLoader.ANALYZE_STRUCTURE); - - transientState.forceHeader = Chars.equalsNc("true", rh.getUrlParam("forceHeader")); - transientState.messagePart = MESSAGE_DATA; - } else if (Chars.equalsNc("schema", contentDisposition)) { - transientState.textLoader.setState(TextLoader.LOAD_JSON_METADATA); - transientState.messagePart = MESSAGE_SCHEMA; - } else { - if (partHeader.getContentDisposition() == null) { - transientContext.simpleResponse().sendStatus(400, "'Content-Disposition' multipart header missing'"); - } else { - transientContext.simpleResponse().sendStatus(400, "invalid value in 'Content-Disposition' multipart header"); - } - throw ServerDisconnectException.INSTANCE; - } - } - - @Override - public void onPartEnd() throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { - try { - LOG.debug().$("part end").$(); - transientState.textLoader.wrapUp(); - if (transientState.messagePart == MESSAGE_DATA) { - sendResponse(transientContext); - } - } catch (TextException | CairoException | CairoError e) { - handleTextException(e); - } - } - - @Override - public void onRequestComplete(HttpConnectionContext context) { - transientState.clear(); - } - - @Override - public void resumeRecv(HttpConnectionContext context) { - this.transientContext = context; - this.transientState = LV.get(context); - if (this.transientState == null) { - LOG.debug().$("new text state").$(); - LV.set(context, this.transientState = new TextImportProcessorState(engine)); - transientState.json = Chars.equalsNc("json", context.getRequestHeader().getUrlParam("fmt")); - } - } - - @Override - public void resumeSend( - HttpConnectionContext context - ) throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { - doResumeSend(LV.get(context), context.getChunkedResponseSocket()); - } - private void doResumeSend( TextImportProcessorState state, HttpChunkedResponseSocket socket @@ -392,10 +387,18 @@ public class TextImportProcessor implements HttpRequestProcessor, HttpMultipartC private void handleTextException(FlyweightMessageContainer e) throws PeerDisconnectedException, PeerIsSlowToReadException, ServerDisconnectException { - sendError(transientContext, e.getFlyweightMessage(), Chars.equalsNc("json", transientContext.getRequestHeader().getUrlParam("fmt"))); + sendError(transientContext, e.getFlyweightMessage(), isJson(transientContext)); throw ServerDisconnectException.INSTANCE; } + private boolean isJson(HttpConnectionContext transientContext) { + return Chars.equalsNc("json", transientContext.getRequestHeader().getUrlParam("fmt")); + } + + private void sendError(CharSequence message) throws PeerDisconnectedException, PeerIsSlowToReadException { + sendError(transientContext, message, transientState.json); + } + private void sendError( HttpConnectionContext context, CharSequence message, @@ -403,11 +406,11 @@ public class TextImportProcessor implements HttpRequestProcessor, HttpMultipartC ) throws PeerDisconnectedException, PeerIsSlowToReadException { final HttpChunkedResponseSocket socket = context.getChunkedResponseSocket(); if (json) { - socket.status(400, CONTENT_TYPE_JSON); + socket.status(200, CONTENT_TYPE_JSON); socket.sendHeader(); socket.put('{').putQuoted("status").put(':').encodeUtf8AndQuote(message).put('}'); } else { - socket.status(400, CONTENT_TYPE_TEXT); + socket.status(200, CONTENT_TYPE_TEXT); socket.sendHeader(); socket.encodeUtf8(message); } @@ -431,4 +434,9 @@ public class TextImportProcessor implements HttpRequestProcessor, HttpMultipartC sendError(context, state.stateMessage, state.json); } } + + static { + atomicityParamMap.put("skipRow", Atomicity.SKIP_ROW); + atomicityParamMap.put("abort", Atomicity.SKIP_ALL); + } } diff --git a/core/src/main/java/io/questdb/cutlass/http/processors/TextQueryProcessor.java b/core/src/main/java/io/questdb/cutlass/http/processors/TextQueryProcessor.java index 029b4742405bf2f0cf1239f1a667a0959472596d..448dc1d53fe3f9aa7dca043d5522ef704d172642 100644 --- a/core/src/main/java/io/questdb/cutlass/http/processors/TextQueryProcessor.java +++ b/core/src/main/java/io/questdb/cutlass/http/processors/TextQueryProcessor.java @@ -125,7 +125,7 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { try { state.cursor = state.recordCursorFactory.getCursor(sqlExecutionContext); state.metadata = state.recordCursorFactory.getMetadata(); - header(context.getChunkedResponseSocket(), 200); + header(context.getChunkedResponseSocket()); resumeSend(context); } catch (CairoException e) { state.setQueryCacheable(e.isCacheable()); @@ -134,7 +134,7 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { internalError(context.getChunkedResponseSocket(), e, state); } } else { - header(context.getChunkedResponseSocket(), 200); + header(context.getChunkedResponseSocket()); sendConfirmation(context.getChunkedResponseSocket()); readyForNextRequest(context); } @@ -299,11 +299,8 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { return LOG.error().$('[').$(state.getFd()).$("] "); } - protected void header( - HttpChunkedResponseSocket socket, - int status - ) throws PeerDisconnectedException, PeerIsSlowToReadException { - socket.status(status, "text/csv; charset=utf-8"); + protected void header(HttpChunkedResponseSocket socket) throws PeerDisconnectedException, PeerIsSlowToReadException { + socket.status(200, "text/csv; charset=utf-8"); socket.headers().put("Content-Disposition: attachment; filename=\"questdb-query-").put(clock.getTicks()).put(".csv\"").put(Misc.EOL); socket.headers().setKeepAlive(configuration.getKeepAliveHeader()); socket.sendHeader(); @@ -319,7 +316,7 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { TextQueryProcessorState state ) throws PeerDisconnectedException, PeerIsSlowToReadException { error(state).$("Server error executing query ").utf8(state.query).$(e).$(); - sendException(socket, 0, e.getMessage(), 500, state.query); + sendException(socket, 0, e.getMessage(), state.query); } private boolean parseUrl( @@ -331,7 +328,7 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { final DirectByteCharSequence query = request.getUrlParam("query"); if (query == null || query.length() == 0) { info(state).$("Empty query request received. Sending empty reply.").$(); - sendException(socket, 0, "No query text", 400, state.query); + sendException(socket, 0, "No query text", state.query); return false; } @@ -372,7 +369,7 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { TextUtil.utf8Decode(query.getLo(), query.getHi(), state.query); } catch (Utf8Exception e) { info(state).$("Bad UTF8 encoding").$(); - sendException(socket, 0, "Bad UTF8 encoding in query text", 400, state.query); + sendException(socket, 0, "Bad UTF8 encoding in query text", state.query); return false; } state.skip = skip; @@ -473,17 +470,10 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { HttpChunkedResponseSocket socket, int position, CharSequence message, - int status, CharSequence query ) throws PeerDisconnectedException, PeerIsSlowToReadException { - header(socket, status); - socket.put('{'). - putQuoted("query").put(':').encodeUtf8AndQuote(query == null ? "" : query).put(','). - putQuoted("error").put(':').encodeUtf8AndQuote(message).put(','). - putQuoted("position").put(':').put(position); - socket.put('}'); - socket.sendChunk(); - socket.done(); + header(socket); + JsonQueryProcessorState.prepareExceptionJson(socket, position, message, query); } private void syntaxError( @@ -496,6 +486,6 @@ public class TextQueryProcessor implements HttpRequestProcessor, Closeable { .$("`, at=").$(sqlException.getPosition()) .$(", message=`").$(sqlException.getFlyweightMessage()).$('`') .$(']').$(); - sendException(socket, sqlException.getPosition(), sqlException.getFlyweightMessage(), 400, state.query); + sendException(socket, sqlException.getPosition(), sqlException.getFlyweightMessage(), state.query); } } diff --git a/core/src/test/java/io/questdb/cutlass/http/IODispatcherTest.java b/core/src/test/java/io/questdb/cutlass/http/IODispatcherTest.java index f375a077c37a07210d28217c6245bf5375b3f573..2761544de8835e2486ca17fe894a8a9f6f095847 100644 --- a/core/src/test/java/io/questdb/cutlass/http/IODispatcherTest.java +++ b/core/src/test/java/io/questdb/cutlass/http/IODispatcherTest.java @@ -272,11 +272,11 @@ public class IODispatcherTest { "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "Cookie: _ga=GA1.1.2124932001.1573824669; _gid=GA1.1.1731187971.1580598042\r\n" + "\r\n", - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "14\r\n" + "table name missing\r\n" + @@ -369,7 +369,7 @@ public class IODispatcherTest { "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "08\r\n" + "Exists\r\n" + @@ -590,7 +590,7 @@ public class IODispatcherTest { @Test public void testImportBadJson() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -652,80 +652,14 @@ public class IODispatcherTest { ); } - @Test - public void testPostRequestToGetProcessor() throws Exception { - testImport( - "HTTP/1.1 400 Bad request\r\n" + - "Server: questDB/1.0\r\n" + - "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + - "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + - "\r\n" + - "2a\r\n" + - "Bad request. non-multipart GET expected.\r\n" + - "\r\n" + - "00\r\n" + - "\r\n", - "POST /exec?fmt=json&overwrite=true&forceHeader=true&name=clipboard-157200856 HTTP/1.1\r\n" + - "Host: localhost:9001\r\n" + - "Connection: keep-alive\r\n" + - "Content-Length: 832\r\n" + - "Accept: */*\r\n" + - "Origin: http://localhost:9000\r\n" + - "X-Requested-With: XMLHttpRequest\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36\r\n" + - "Sec-Fetch-Mode: cors\r\n" + - "Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryOsOAD9cPKyHuxyBV\r\n" + - "Sec-Fetch-Site: same-origin\r\n" + - "Referer: http://localhost:9000/index.html\r\n" + - "Accept-Encoding: gzip, deflate, br\r\n" + - "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + - "\r\n" + - "------WebKitFormBoundaryOsOAD9cPKyHuxyBV\r\n" + - "Content-Disposition: form-data; name=\"schema\"\r\n" + - "\r\n" + - "[{\"name\":\"timestamp,\"type\":\"DATE\"},{\"name\":\"bid\",\"type\":\"INT\"}]\r\n" + - "------WebKitFormBoundaryOsOAD9cPKyHuxyBV\r\n" + - "Content-Disposition: form-data; name=\"data\"\r\n" + - "\r\n" + - "timestamp,bid\r\n" + - "27/05/2018 00:00:01,100\r\n" + - "27/05/2018 00:00:02,101\r\n" + - "27/05/2018 00:00:03,102\r\n" + - "27/05/2018 00:00:04,103\r\n" + - "27/05/2018 00:00:05,104\r\n" + - "27/05/2018 00:00:06,105\r\n" + - "27/05/2018 00:00:07,106\r\n" + - "27/05/2018 00:00:08,107\r\n" + - "27/05/2018 00:00:09,108\r\n" + - "27/05/2018 00:00:10,109\r\n" + - "27/05/2018 00:00:11,110\r\n" + - "27/05/2018 00:00:12,111\r\n" + - "27/05/2018 00:00:13,112\r\n" + - "27/05/2018 00:00:14,113\r\n" + - "27/05/2018 00:00:15,114\r\n" + - "27/05/2018 00:00:16,115\r\n" + - "27/05/2018 00:00:17,116\r\n" + - "27/05/2018 00:00:18,117\r\n" + - "27/05/2018 00:00:19,118\r\n" + - "27/05/2018 00:00:20,119\r\n" + - "27/05/2018 00:00:21,120\r\n" + - "\r\n" + - "------WebKitFormBoundaryOsOAD9cPKyHuxyBV--", - NetworkFacadeImpl.INSTANCE, - true, - 1 - ); - } - @Test public void testImportBadRequestGet() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 404 Not Found\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "27\r\n" + "Bad request. Multipart POST expected.\r\n" + @@ -839,7 +773,7 @@ public class IODispatcherTest { // append different data structure to the same table testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -911,7 +845,7 @@ public class IODispatcherTest { @Test public void testImportDelimiterNotDetected() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -986,7 +920,7 @@ public class IODispatcherTest { @Test public void testImportForceUnknownDate() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -1051,7 +985,7 @@ public class IODispatcherTest { @Test public void testImportForceUnknownTimestamp() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -1845,7 +1779,7 @@ public class IODispatcherTest { "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -1888,22 +1822,21 @@ public class IODispatcherTest { } @Test - public void testJsonQueryCreateInsertTruncateAndDrop() throws Exception { + public void testJsonQueryCreateInsertStringifiedJson() throws Exception { testJsonQuery0(1, engine -> { - // create table sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?query=%0A%0A%0Acreate+table+balances_x+(%0A%09cust_id+int%2C+%0A%09balance_ccy+symbol%2C+%0A%09balance+double%2C+%0A%09status+byte%2C+%0A%09timestamp+timestamp%0A)&limit=0%2C1000&count=true HTTP/1.1\r\n" + - "Host: localhost:9000\r\n" + + "GET /query?limit=0%2C1000&count=true&src=con&query=%0D%0Acreate%20table%20data(s%20string)&timings=true HTTP/1.1\r\n" + + "Host: 127.0.0.1:9000\r\n" + "Connection: keep-alive\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36\r\n" + "Accept: */*\r\n" + - "X-Requested-With: XMLHttpRequest\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + - "Referer: http://localhost:9000/index.html\r\n" + - "Accept-Encoding: gzip, deflate, br\r\n" + + "Sec-Fetch-Dest: empty\r\n" + + "Referer: http://127.0.0.1:9000/\r\n" + + "Accept-Encoding: gzip, deflate, br\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", "HTTP/1.1 200 OK\r\n" + @@ -1925,15 +1858,15 @@ public class IODispatcherTest { // insert one record sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?query=%0A%0Ainsert+into+balances_x+(cust_id%2C+balance_ccy%2C+balance%2C+timestamp)+values+(1%2C+%27USD%27%2C+1500.00%2C+6000000001)&limit=0%2C1000&count=true HTTP/1.1\r\n" + - "Host: localhost:9000\r\n" + + "GET /query?limit=0%2C1000&count=true&src=con&query=%0D%0A%0D%0Ainsert%20into%20data%20values%20(%27%7B%20title%3A%20%5C%22Title%5C%22%7D%27)&timings=true HTTP/1.1\r\n" + + "Host: 127.0.0.1:9000\r\n" + "Connection: keep-alive\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36\r\n" + "Accept: */*\r\n" + - "X-Requested-With: XMLHttpRequest\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + - "Referer: http://localhost:9000/index.html\r\n" + + "Sec-Fetch-Dest: empty\r\n" + + "Referer: http://127.0.0.1:9000/\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", @@ -1956,15 +1889,15 @@ public class IODispatcherTest { // check if we have one record sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?query=%0A%0Aselect+*+from+balances_x+latest+by+cust_id%2C+balance_ccy&limit=0%2C1000&count=true HTTP/1.1\r\n" + - "Host: localhost:9000\r\n" + + "GET /query?limit=0%2C1000&count=true&src=con&query=data&timings=false HTTP/1.1\r\n" + + "Host: 127.0.0.1:9000\r\n" + "Connection: keep-alive\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36\r\n" + "Accept: */*\r\n" + - "X-Requested-With: XMLHttpRequest\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + - "Referer: http://localhost:9000/index.html\r\n" + + "Sec-Fetch-Dest: empty\r\n" + + "Referer: http://127.0.0.1:9000/\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", @@ -1975,19 +1908,25 @@ public class IODispatcherTest { "Content-Type: application/json; charset=utf-8\r\n" + "Keep-Alive: timeout=5, max=10000\r\n" + "\r\n" + - "014c\r\n" + - "{\"query\":\"\\n\\nselect * from balances_x latest by cust_id, balance_ccy\",\"columns\":[{\"name\":\"cust_id\",\"type\":\"INT\"},{\"name\":\"balance_ccy\",\"type\":\"SYMBOL\"},{\"name\":\"balance\",\"type\":\"DOUBLE\"},{\"name\":\"status\",\"type\":\"BYTE\"},{\"name\":\"timestamp\",\"type\":\"TIMESTAMP\"}],\"dataset\":[[1,\"USD\",1500.0,0,\"1970-01-01T01:40:00.000001Z\"]],\"count\":1}\r\n" + + "6b\r\n" + + "{\"query\":\"data\",\"columns\":[{\"name\":\"s\",\"type\":\"STRING\"}],\"dataset\":[[\"{ title: \\\\\\\"Title\\\\\\\"}\"]],\"count\":1}\r\n" + "00\r\n" + "\r\n", 1, 0, false ); + }, false); + } - // truncate table + @Test + public void testJsonQueryCreateInsertTruncateAndDrop() throws Exception { + testJsonQuery0(1, engine -> { + + // create table sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?query=%0A%0Atruncate+table+balances_x&limit=0%2C1000&count=true HTTP/1.1\r\n" + + "GET /query?query=%0A%0A%0Acreate+table+balances_x+(%0A%09cust_id+int%2C+%0A%09balance_ccy+symbol%2C+%0A%09balance+double%2C+%0A%09status+byte%2C+%0A%09timestamp+timestamp%0A)&limit=0%2C1000&count=true HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Accept: */*\r\n" + @@ -2015,10 +1954,10 @@ public class IODispatcherTest { false ); - // select again expecting only metadata + // insert one record sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?query=%0A%0Aselect+*+from+balances_x+latest+by+cust_id%2C+balance_ccy&limit=0%2C1000&count=true HTTP/1.1\r\n" + + "GET /query?query=%0A%0Ainsert+into+balances_x+(cust_id%2C+balance_ccy%2C+balance%2C+timestamp)+values+(1%2C+%27USD%27%2C+1500.00%2C+6000000001)&limit=0%2C1000&count=true HTTP/1.1\r\n" + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + "Accept: */*\r\n" + @@ -2037,33 +1976,28 @@ public class IODispatcherTest { "Content-Type: application/json; charset=utf-8\r\n" + "Keep-Alive: timeout=5, max=10000\r\n" + "\r\n" + - "011c\r\n" + - "{\"query\":\"\\n\\nselect * from balances_x latest by cust_id, balance_ccy\",\"columns\":[{\"name\":\"cust_id\",\"type\":\"INT\"},{\"name\":\"balance_ccy\",\"type\":\"SYMBOL\"},{\"name\":\"balance\",\"type\":\"DOUBLE\"},{\"name\":\"status\",\"type\":\"BYTE\"},{\"name\":\"timestamp\",\"type\":\"TIMESTAMP\"}],\"dataset\":[],\"count\":0}\r\n" + + "0c\r\n" + + "{\"ddl\":\"OK\"}\r\n" + "00\r\n" + "\r\n", 1, 0, false ); - }, false); - } - @Test - public void testJsonQueryCreateInsertStringifiedJson() throws Exception { - testJsonQuery0(1, engine -> { - // create table + // check if we have one record sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?limit=0%2C1000&count=true&src=con&query=%0D%0Acreate%20table%20data(s%20string)&timings=true HTTP/1.1\r\n" + - "Host: 127.0.0.1:9000\r\n" + + "GET /query?query=%0A%0Aselect+*+from+balances_x+latest+by+cust_id%2C+balance_ccy&limit=0%2C1000&count=true HTTP/1.1\r\n" + + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36\r\n" + "Accept: */*\r\n" + + "X-Requested-With: XMLHttpRequest\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + - "Sec-Fetch-Dest: empty\r\n" + - "Referer: http://127.0.0.1:9000/\r\n" + - "Accept-Encoding: gzip, deflate, br\n" + + "Referer: http://localhost:9000/index.html\r\n" + + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", "HTTP/1.1 200 OK\r\n" + @@ -2073,8 +2007,8 @@ public class IODispatcherTest { "Content-Type: application/json; charset=utf-8\r\n" + "Keep-Alive: timeout=5, max=10000\r\n" + "\r\n" + - "0c\r\n" + - "{\"ddl\":\"OK\"}\r\n" + + "014c\r\n" + + "{\"query\":\"\\n\\nselect * from balances_x latest by cust_id, balance_ccy\",\"columns\":[{\"name\":\"cust_id\",\"type\":\"INT\"},{\"name\":\"balance_ccy\",\"type\":\"SYMBOL\"},{\"name\":\"balance\",\"type\":\"DOUBLE\"},{\"name\":\"status\",\"type\":\"BYTE\"},{\"name\":\"timestamp\",\"type\":\"TIMESTAMP\"}],\"dataset\":[[1,\"USD\",1500.0,0,\"1970-01-01T01:40:00.000001Z\"]],\"count\":1}\r\n" + "00\r\n" + "\r\n", 1, @@ -2082,18 +2016,18 @@ public class IODispatcherTest { false ); - // insert one record + // truncate table sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?limit=0%2C1000&count=true&src=con&query=%0D%0A%0D%0Ainsert%20into%20data%20values%20(%27%7B%20title%3A%20%5C%22Title%5C%22%7D%27)&timings=true HTTP/1.1\r\n" + - "Host: 127.0.0.1:9000\r\n" + + "GET /query?query=%0A%0Atruncate+table+balances_x&limit=0%2C1000&count=true HTTP/1.1\r\n" + + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36\r\n" + "Accept: */*\r\n" + + "X-Requested-With: XMLHttpRequest\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + - "Sec-Fetch-Dest: empty\r\n" + - "Referer: http://127.0.0.1:9000/\r\n" + + "Referer: http://localhost:9000/index.html\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", @@ -2113,18 +2047,18 @@ public class IODispatcherTest { false ); - // check if we have one record + // select again expecting only metadata sendAndReceive( NetworkFacadeImpl.INSTANCE, - "GET /query?limit=0%2C1000&count=true&src=con&query=data&timings=false HTTP/1.1\r\n" + - "Host: 127.0.0.1:9000\r\n" + + "GET /query?query=%0A%0Aselect+*+from+balances_x+latest+by+cust_id%2C+balance_ccy&limit=0%2C1000&count=true HTTP/1.1\r\n" + + "Host: localhost:9000\r\n" + "Connection: keep-alive\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36\r\n" + "Accept: */*\r\n" + + "X-Requested-With: XMLHttpRequest\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: cors\r\n" + - "Sec-Fetch-Dest: empty\r\n" + - "Referer: http://127.0.0.1:9000/\r\n" + + "Referer: http://localhost:9000/index.html\r\n" + "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", @@ -2135,8 +2069,8 @@ public class IODispatcherTest { "Content-Type: application/json; charset=utf-8\r\n" + "Keep-Alive: timeout=5, max=10000\r\n" + "\r\n" + - "6b\r\n" + - "{\"query\":\"data\",\"columns\":[{\"name\":\"s\",\"type\":\"STRING\"}],\"dataset\":[[\"{ title: \\\\\\\"Title\\\\\\\"}\"]],\"count\":1}\r\n" + + "011c\r\n" + + "{\"query\":\"\\n\\nselect * from balances_x latest by cust_id, balance_ccy\",\"columns\":[{\"name\":\"cust_id\",\"type\":\"INT\"},{\"name\":\"balance_ccy\",\"type\":\"SYMBOL\"},{\"name\":\"balance\",\"type\":\"DOUBLE\"},{\"name\":\"status\",\"type\":\"BYTE\"},{\"name\":\"timestamp\",\"type\":\"TIMESTAMP\"}],\"dataset\":[],\"count\":0}\r\n" + "00\r\n" + "\r\n", 1, @@ -2340,7 +2274,7 @@ public class IODispatcherTest { "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -2366,7 +2300,7 @@ public class IODispatcherTest { "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -2423,7 +2357,7 @@ public class IODispatcherTest { "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -2447,7 +2381,7 @@ public class IODispatcherTest { "Accept-Encoding: gzip, deflate, br\r\n" + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "\r\n", - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -2514,7 +2448,7 @@ public class IODispatcherTest { "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + "Cookie: _ga=GA1.1.2124932001.1573824669; _gid=GA1.1.1731187971.1580598042\r\n" + "\r\n", - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + @@ -3131,12 +3065,12 @@ public class IODispatcherTest { final String expectedEvent = "100\n" + "1\n" + "101\n"; - assertColumn(expectedEvent, "telemetry", 1); + assertColumn(expectedEvent, 1); final String expectedOrigin = "1\n" + "2\n" + "1\n"; - assertColumn(expectedOrigin, "telemetry", 2); + assertColumn(expectedOrigin, 2); } @Test @@ -3172,13 +3106,13 @@ public class IODispatcherTest { "1\n" + "1\n" + "101\n"; - assertColumn(expected, "telemetry", 1); + assertColumn(expected, 1); final String expectedOrigin = "1\n" + "2\n" + "2\n" + "1\n"; - assertColumn(expectedOrigin, "telemetry", 2); + assertColumn(expectedOrigin, 2); } @Test @@ -3238,47 +3172,50 @@ public class IODispatcherTest { workerPool.start(LOG); - // create table with all column types - CairoTestUtils.createTestTable( - engine.getConfiguration(), - 20, - new Rnd(), - new TestRecord.ArrayBinarySequence()); + try { - // send multipart request to server - final String request = "GET /query?query=x%20where2%20i%20%3D%20(%27EHNRX%27) HTTP/1.1\r\n" + - "Host: localhost:9001\r\n" + - "Connection: keep-alive\r\n" + - "Cache-Control: max-age=0\r\n" + - "Upgrade-Insecure-Requests: 1\r\n" + - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + - "Accept-Encoding: gzip, deflate, br\r\n" + - "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + - "\r\n"; + // create table with all column types + CairoTestUtils.createTestTable( + engine.getConfiguration(), + 20, + new Rnd(), + new TestRecord.ArrayBinarySequence()); - String expectedResponse = "HTTP/1.1 400 Bad request\r\n" + - "Server: questDB/1.0\r\n" + - "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + - "Transfer-Encoding: chunked\r\n" + - "Content-Type: application/json; charset=utf-8\r\n" + - "Keep-Alive: timeout=5, max=10000\r\n" + - "\r\n" + - "4d\r\n" + - "{\"query\":\"x where2 i = ('EHNRX')\",\"error\":\"unexpected token: i\",\"position\":9}\r\n" + - "00\r\n" + - "\r\n"; + // send multipart request to server + final String request = "GET /query?query=x%20where2%20i%20%3D%20(%27EHNRX%27) HTTP/1.1\r\n" + + "Host: localhost:9001\r\n" + + "Connection: keep-alive\r\n" + + "Cache-Control: max-age=0\r\n" + + "Upgrade-Insecure-Requests: 1\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3\r\n" + + "Accept-Encoding: gzip, deflate, br\r\n" + + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + + "\r\n"; - sendAndReceive( - NetworkFacadeImpl.INSTANCE, - request, - expectedResponse, - 10, - 0, - false - ); + String expectedResponse = "HTTP/1.1 200 OK\r\n" + + "Server: questDB/1.0\r\n" + + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Type: application/json; charset=utf-8\r\n" + + "Keep-Alive: timeout=5, max=10000\r\n" + + "\r\n" + + "4d\r\n" + + "{\"query\":\"x where2 i = ('EHNRX')\",\"error\":\"unexpected token: i\",\"position\":9}\r\n" + + "00\r\n" + + "\r\n"; - workerPool.halt(); + sendAndReceive( + NetworkFacadeImpl.INSTANCE, + request, + expectedResponse, + 10, + 0, + false + ); + } finally { + workerPool.halt(); + } } }); } @@ -3696,15 +3633,14 @@ public class IODispatcherTest { @Test public void testMissingContentDisposition() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + - "31\r\n" + + "2f\r\n" + "'Content-Disposition' multipart header missing'\r\n" + - "\r\n" + "00\r\n" + "\r\n", "POST /upload HTTP/1.1\r\n" + @@ -3730,15 +3666,14 @@ public class IODispatcherTest { @Test public void testMissingContentDispositionFileName() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + - "14\r\n" + + "12\r\n" + "no file name given\r\n" + - "\r\n" + "00\r\n" + "\r\n", "POST /upload HTTP/1.1\r\n" + @@ -3765,15 +3700,14 @@ public class IODispatcherTest { @Test public void testMissingContentDispositionName() throws Exception { testImport( - "HTTP/1.1 400 Bad request\r\n" + + "HTTP/1.1 200 OK\r\n" + "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + - "39\r\n" + + "37\r\n" + "invalid value in 'Content-Disposition' multipart header\r\n" + - "\r\n" + "00\r\n" + "\r\n", "POST /upload HTTP/1.1\r\n" + @@ -3856,6 +3790,72 @@ public class IODispatcherTest { }, false); } + @Test + public void testPostRequestToGetProcessor() throws Exception { + testImport( + "HTTP/1.1 404 Not Found\r\n" + + "Server: questDB/1.0\r\n" + + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + + "\r\n" + + "2a\r\n" + + "Bad request. non-multipart GET expected.\r\n" + + "\r\n" + + "00\r\n" + + "\r\n", + "POST /exec?fmt=json&overwrite=true&forceHeader=true&name=clipboard-157200856 HTTP/1.1\r\n" + + "Host: localhost:9001\r\n" + + "Connection: keep-alive\r\n" + + "Content-Length: 832\r\n" + + "Accept: */*\r\n" + + "Origin: http://localhost:9000\r\n" + + "X-Requested-With: XMLHttpRequest\r\n" + + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36\r\n" + + "Sec-Fetch-Mode: cors\r\n" + + "Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryOsOAD9cPKyHuxyBV\r\n" + + "Sec-Fetch-Site: same-origin\r\n" + + "Referer: http://localhost:9000/index.html\r\n" + + "Accept-Encoding: gzip, deflate, br\r\n" + + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8\r\n" + + "\r\n" + + "------WebKitFormBoundaryOsOAD9cPKyHuxyBV\r\n" + + "Content-Disposition: form-data; name=\"schema\"\r\n" + + "\r\n" + + "[{\"name\":\"timestamp,\"type\":\"DATE\"},{\"name\":\"bid\",\"type\":\"INT\"}]\r\n" + + "------WebKitFormBoundaryOsOAD9cPKyHuxyBV\r\n" + + "Content-Disposition: form-data; name=\"data\"\r\n" + + "\r\n" + + "timestamp,bid\r\n" + + "27/05/2018 00:00:01,100\r\n" + + "27/05/2018 00:00:02,101\r\n" + + "27/05/2018 00:00:03,102\r\n" + + "27/05/2018 00:00:04,103\r\n" + + "27/05/2018 00:00:05,104\r\n" + + "27/05/2018 00:00:06,105\r\n" + + "27/05/2018 00:00:07,106\r\n" + + "27/05/2018 00:00:08,107\r\n" + + "27/05/2018 00:00:09,108\r\n" + + "27/05/2018 00:00:10,109\r\n" + + "27/05/2018 00:00:11,110\r\n" + + "27/05/2018 00:00:12,111\r\n" + + "27/05/2018 00:00:13,112\r\n" + + "27/05/2018 00:00:14,113\r\n" + + "27/05/2018 00:00:15,114\r\n" + + "27/05/2018 00:00:16,115\r\n" + + "27/05/2018 00:00:17,116\r\n" + + "27/05/2018 00:00:18,117\r\n" + + "27/05/2018 00:00:19,118\r\n" + + "27/05/2018 00:00:20,119\r\n" + + "27/05/2018 00:00:21,120\r\n" + + "\r\n" + + "------WebKitFormBoundaryOsOAD9cPKyHuxyBV--", + NetworkFacadeImpl.INSTANCE, + true, + 1 + ); + } + @Test public void testSCPConnectDownloadDisconnect() throws Exception { assertMemoryLeak(() -> { @@ -4000,7 +4000,7 @@ public class IODispatcherTest { "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + @@ -4009,40 +4009,17 @@ public class IODispatcherTest { "\r\n"; - for (int i = 0; i < 4; i++) { - long fd = Net.socketTcp(true); - Assert.assertTrue(fd > -1); - Assert.assertEquals(0, Net.connect(fd, sockAddr)); - try { - sendRequest(request3, fd, buffer); - assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader3, expectedResponseHeader3.length()); - } finally { - Net.close(fd); - } - } - + sendAndReceive(NetworkFacadeImpl.INSTANCE, request3, expectedResponseHeader3, 4, 0, false); // and few more 304s - - for (int i = 0; i < 3; i++) { - long fd = Net.socketTcp(true); - Assert.assertTrue(fd > -1); - Assert.assertEquals(0, Net.connect(fd, sockAddr)); - try { - sendRequest(request2, fd, buffer); - assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126); - } finally { - Net.close(fd); - } - } - + sendAndReceive(NetworkFacadeImpl.INSTANCE, request2, expectedResponseHeader2, 4, 0, false); } finally { Unsafe.free(buffer, netBufferLen); } } finally { Net.freeSockAddr(sockAddr); } - workerPool.halt(); } finally { + workerPool.halt(); Files.remove(path); } } @@ -4092,7 +4069,7 @@ public class IODispatcherTest { Rnd rnd = new Rnd(); final int diskBufferLen = 1024 * 1024; - writeRandomFile(path, rnd, 122222212222L, diskBufferLen); + writeRandomFile(path, rnd, 122299092L, diskBufferLen); long fd = Net.socketTcp(true); try { @@ -4122,12 +4099,12 @@ public class IODispatcherTest { "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Content-Length: 20971520\r\n" + "Content-Type: text/plain\r\n" + - "ETag: \"122222212222\"\r\n" + // this is last modified timestamp on the file, we set this value when we created file + "ETag: \"122299092\"\r\n" + // this is last modified timestamp on the file, we set this value when we created file "\r\n"; for (int j = 0; j < 10; j++) { sendRequest(request, fd, buffer); - assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670); + assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971667); } // // send few requests to receive 304 @@ -4139,7 +4116,7 @@ public class IODispatcherTest { "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36\r\n" + "Accept-Encoding: gzip,deflate,sdch\r\n" + "Accept-Language: en-US,en;q=0.8\r\n" + - "If-None-Match: \"122222212222\"\r\n" + // this header should make static processor return 304 + "If-None-Match: \"122299092\"\r\n" + // this header should make static processor return 304 "Cookie: textwrapon=false; textautoformat=false; wysiwyg=textarea\r\n" + "\r\n"; @@ -4157,7 +4134,7 @@ public class IODispatcherTest { // couple more full downloads after 304 for (int j = 0; j < 2; j++) { sendRequest(request, fd, buffer); - assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971670); + assertDownloadResponse(fd, rnd, buffer, netBufferLen, diskBufferLen, expectedResponseHeader, 20971667); } // get a 404 now @@ -4176,7 +4153,7 @@ public class IODispatcherTest { "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "0b\r\n" + "Not Found\r\n" + @@ -4185,18 +4162,9 @@ public class IODispatcherTest { "\r\n"; - for (int i = 0; i < 4; i++) { - sendRequest(request3, fd, buffer); - assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader3, expectedResponseHeader3.length()); - } - + sendAndReceive(NetworkFacadeImpl.INSTANCE, request3, expectedResponseHeader3, 4, 0, false); // and few more 304s - - for (int i = 0; i < 3; i++) { - sendRequest(request2, fd, buffer); - assertDownloadResponse(fd, rnd, buffer, netBufferLen, 0, expectedResponseHeader2, 126); - } - + sendAndReceive(NetworkFacadeImpl.INSTANCE, request2, expectedResponseHeader2, 4, 0, false); } finally { Unsafe.free(buffer, netBufferLen); } @@ -4207,9 +4175,8 @@ public class IODispatcherTest { Net.close(fd); LOG.info().$("closed [fd=").$(fd).$(']').$(); } - - workerPool.halt(); } finally { + workerPool.halt(); Files.remove(path); } } @@ -4363,7 +4330,7 @@ public class IODispatcherTest { "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n" + "0b\r\n" + @@ -4594,7 +4561,7 @@ public class IODispatcherTest { "Server: questDB/1.0\r\n" + "Date: Thu, 1 Jan 1970 00:00:00 GMT\r\n" + "Transfer-Encoding: chunked\r\n" + - "Content-Type: text/html; charset=utf-8\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + "\r\n" + "04\r\n" + "OK\r\n" + @@ -5155,6 +5122,7 @@ public class IODispatcherTest { if (n > 0) { if (headerCheckRemaining > 0) { for (int i = 0; i < n && headerCheckRemaining > 0; i++) { +// System.out.print(expectedResponseHeader.charAt(expectedHeaderLen - headerCheckRemaining)); if (expectedResponseHeader.charAt(expectedHeaderLen - headerCheckRemaining) != (char) Unsafe.getUnsafe().getByte(buffer + i)) { Assert.fail("at " + (expectedHeaderLen - headerCheckRemaining)); } @@ -5185,11 +5153,11 @@ public class IODispatcherTest { Assert.assertEquals(requestLen, Net.send(fd, buffer, requestLen)); } - private void assertColumn(CharSequence expected, CharSequence tableName, int index) { + private void assertColumn(CharSequence expected, int index) { final String baseDir = temp.getRoot().getAbsolutePath(); DefaultCairoConfiguration configuration = new DefaultCairoConfiguration(baseDir); - try (TableReader reader = new TableReader(configuration, tableName)) { + try (TableReader reader = new TableReader(configuration, "telemetry")) { final StringSink sink = new StringSink(); final RecordCursorPrinter printer = new RecordCursorPrinter(sink); sink.clear(); @@ -5431,8 +5399,10 @@ public class IODispatcherTest { final int expectedToReceive = expectedResponse.length; int received = 0; if (print) { - System.out.println("expected"); - System.out.println(new String(expectedResponse, StandardCharsets.UTF_8)); + System.out.println("expected=`"); + System.out.print(new String(expectedResponse, StandardCharsets.UTF_8)); + System.out.println('`'); + System.out.println("received=`"); } boolean disconnected = false; while (received < expectedToReceive) { @@ -5459,6 +5429,9 @@ public class IODispatcherTest { break; } } + if (print) { + System.out.println('`'); + } if (disconnected && !expectDisconnect) { LOG.error().$("disconnected?").$(); Assert.fail(); diff --git a/ui/src/utils/questdb.ts b/ui/src/utils/questdb.ts index d2992bc80da431c2f63f3c7f427ea0f4525443fd..44c3e63c7ec88c288973997ca1e65c15696ff6ae 100644 --- a/ui/src/utils/questdb.ts +++ b/ui/src/utils/questdb.ts @@ -26,6 +26,7 @@ type RawDqlResult = { count: number dataset: any[][] ddl: undefined + error: undefined query: string timings: Timings } @@ -35,7 +36,8 @@ type RawDdlResult = { } type RawErrorResult = { - error: string + ddl: undefined + error: "" position: number query: string } @@ -45,7 +47,7 @@ type DdlResult = { type: Type.DDL } -type RawResult = RawDqlResult | RawDdlResult +type RawResult = RawDqlResult | RawDdlResult | RawErrorResult export type ErrorResult = RawErrorResult & { type: Type.ERROR @@ -200,6 +202,14 @@ export class Client { } } + if (data.error) { + // eslint-disable-next-line prefer-promise-reject-errors + return Promise.reject({ + ...data, + type: Type.ERROR, + }) + } + return { ...data, timings: { @@ -210,19 +220,9 @@ export class Client { } } - if (/quest/.test(response.headers.get("server") || "")) { - const data = (await response.json()) as RawErrorResult - - // eslint-disable-next-line prefer-promise-reject-errors - return Promise.reject({ - ...data, - type: Type.ERROR, - }) - } - // eslint-disable-next-line prefer-promise-reject-errors return Promise.reject({ - error: "QuestDB is not reachable", + error: `QuestDB is not reachable [${response.status}]`, position: -1, query, type: Type.ERROR,