Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make some changes to Comm Tasks (#10520) 

* Make some changes to VST handlers

* clear response body after sending

* less code
This commit is contained in:
Simon 2019-11-26 01:01:38 +08:00 committed by KVS85
parent b94755c61e
commit 45cccf8537
11 changed files with 157 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
arangod/Cluster/ClusterComm.h
View File

@ -296,10 +296,10 @@ struct ClusterCommResult {
auto const& headers = response->headers();
auto errorCodes = headers.find(StaticStrings::ErrorCodes);
if (errorCodes != headers.end()) {
request->setHeaderV2(StaticStrings::ErrorCodes, errorCodes->second);
request->setHeaderV2(std::string(StaticStrings::ErrorCodes),
std::string(errorCodes->second));
}
request->setHeaderV2(StaticStrings::ResponseCode,
GeneralResponse::responseString(answer_code));
answer.reset(request);
TRI_ASSERT(response != nullptr);
result = std::make_shared<httpclient::SimpleHttpCommunicatorResult>(

9
arangod/GeneralServer/CommTask.cpp
View File

@ -389,10 +389,11 @@ void CommTask::executeRequest(std::unique_ptr<GeneralRequest> request,
// --SECTION-- statistics handling protected methods
// -----------------------------------------------------------------------------
RequestStatistics* CommTask::acquireStatistics(uint64_t id) {
RequestStatistics* stat = RequestStatistics::acquire();
void CommTask::setStatistics(uint64_t id, RequestStatistics* stat) {
{
std::lock_guard<std::mutex> guard(_statisticsMutex);
if (stat == nullptr) {
auto it = _statisticsMap.find(id);
if (it != _statisticsMap.end()) {
@ -408,10 +409,6 @@ void CommTask::setStatistics(uint64_t id, RequestStatistics* stat) {
}
}
RequestStatistics* CommTask::acquireStatistics(uint64_t id) {
RequestStatistics* stat = RequestStatistics::acquire();
setStatistics(id, stat);
return stat;
}

1
arangod/GeneralServer/CommTask.h
View File

@ -121,7 +121,6 @@ protected:
void executeRequest(std::unique_ptr<GeneralRequest>,
std::unique_ptr<GeneralResponse>);
void setStatistics(uint64_t, RequestStatistics*);
RequestStatistics* acquireStatistics(uint64_t);
RequestStatistics* statistics(uint64_t);
RequestStatistics* stealStatistics(uint64_t);

119
arangod/GeneralServer/HttpCommTask.cpp
View File

@ -82,6 +82,7 @@ int HttpCommTask<T>::on_message_began(llhttp_t* p) {
self->_origin.clear();
self->_request = std::make_unique<HttpRequest>(self->_connectionInfo, /*header*/ nullptr,
0, self->_allowMethodOverride);
self->_response.reset();
self->_lastHeaderWasValue = false;
self->_shouldKeepAlive = false;
self->_messageDone = false;
@ -175,7 +176,9 @@ int HttpCommTask<T>::on_header_complete(llhttp_t* p) {
<< "received a 100-continue request";
char const* response = "HTTP/1.1 100 Continue\r\n\r\n";
auto buff = asio_ns::buffer(response, strlen(response));
asio_ns::async_write(self->_protocol->socket, buff, [self = self->shared_from_this()](asio_ns::error_code const& ec, std::size_t) {
asio_ns::async_write(self->_protocol->socket, buff,
[self = self->shared_from_this()](asio_ns::error_code const& ec,
std::size_t) {
if (ec) {
static_cast<HttpCommTask<T>*>(self.get())->close();
}
@ -237,8 +240,7 @@ HttpCommTask<T>::~HttpCommTask() = default;
template <SocketType T>
void HttpCommTask<T>::start() {
this->_protocol->setNonBlocking(true);
asio_ns::post(this->_protocol->context.io_context,
[self = this->shared_from_this()] {
asio_ns::post(this->_protocol->context.io_context, [self = this->shared_from_this()] {
auto* thisPtr = static_cast<HttpCommTask<T>*>(self.get());
thisPtr->checkVSTPrefix();
});
@ -265,7 +267,6 @@ void HttpCommTask<T>::addSimpleResponse(rest::ResponseCode code,
template <SocketType T>
bool HttpCommTask<T>::readCallback(asio_ns::error_code ec) {
llhttp_errno_t err = HPE_OK;
if (ec) { // got a connection error
if (ec == asio_ns::error::misc_errors::eof) {
@ -327,12 +328,11 @@ bool HttpCommTask<T>::readCallback(asio_ns::error_code ec) {
namespace {
static constexpr const char* vst10 = "VST/1.0\r\n\r\n";
static constexpr const char* vst11 = "VST/1.1\r\n\r\n";
}
} // namespace
template <SocketType T>
void HttpCommTask<T>::checkVSTPrefix() {
auto cb = [self = this->shared_from_this()]
(asio_ns::error_code const& ec, size_t nread) {
auto cb = [self = this->shared_from_this()](asio_ns::error_code const& ec, size_t nread) {
auto* thisPtr = static_cast<HttpCommTask<T>*>(self.get());
if (ec || nread < 11) {
thisPtr->close();
@ -342,7 +342,6 @@ void HttpCommTask<T>::checkVSTPrefix() {
auto bg = asio_ns::buffers_begin(thisPtr->_protocol->buffer.data());
if (std::equal(::vst10, ::vst10 + 11, bg, bg + 11)) {
thisPtr->_protocol->buffer.consume(11); // remove VST/1.0 prefix
auto commTask =
std::make_unique<VstCommTask<T>>(thisPtr->_server, thisPtr->_connectionInfo,
@ -352,7 +351,6 @@ void HttpCommTask<T>::checkVSTPrefix() {
return; // vst 1.0
} else if (std::equal(::vst11, ::vst11 + 11, bg, bg + 11)) {
thisPtr->_protocol->buffer.consume(11); // remove VST/1.1 prefix
auto commTask =
std::make_unique<VstCommTask<T>>(thisPtr->_server, thisPtr->_connectionInfo,
@ -437,8 +435,8 @@ void HttpCommTask<T>::processRequest() {
// unzip / deflate
if (!handleContentEncoding(*_request)) {
this->addErrorResponse(rest::ResponseCode::BAD, _request->contentTypeResponse(), 1,
TRI_ERROR_BAD_PARAMETER, "decoding error");
this->addErrorResponse(rest::ResponseCode::BAD, _request->contentTypeResponse(),
1, TRI_ERROR_BAD_PARAMETER, "decoding error");
return;
}
@ -489,7 +487,6 @@ bool allowCredentials(std::string const& origin) {
}
} // namespace
/// handle an OPTIONS request
template <SocketType T>
void HttpCommTask<T>::processCorsOptions() {
@ -668,13 +665,12 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
response.setHeaderNCIfNotSet(StaticStrings::XContentTypeOptions, StaticStrings::NoSniff);
}
// TODO lease buffers
auto header = std::make_shared<VPackBuffer<uint8_t>>();
header->reserve(220);
_header.clear();
_header.reserve(220);
header->append(TRI_CHAR_LENGTH_PAIR("HTTP/1.1 "));
header->append(GeneralResponse::responseString(response.responseCode()));
header->append("\r\n", 2);
_header.append(TRI_CHAR_LENGTH_PAIR("HTTP/1.1 "));
_header.append(GeneralResponse::responseString(response.responseCode()));
_header.append("\r\n", 2);
bool seenServerHeader = false;
// bool seenConnectionHeader = false;
@ -682,8 +678,7 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
std::string const& key = it.first;
size_t const keyLength = key.size();
// ignore content-length
if (key == StaticStrings::ContentLength ||
key == StaticStrings::Connection ||
if (key == StaticStrings::ContentLength || key == StaticStrings::Connection ||
key == StaticStrings::TransferEncoding) {
continue;
}
@ -693,7 +688,7 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
}
// reserve enough space for header name + ": " + value + "\r\n"
header->reserve(key.size() + 2 + it.second.size() + 2);
_header.reserve(key.size() + 2 + it.second.size() + 2);
char const* p = key.data();
char const* end = p + keyLength;
@ -701,11 +696,11 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
while (p < end) {
if (capState == 1) {
// upper case
header->push_back(StringUtils::toupper(*p));
_header.push_back(StringUtils::toupper(*p));
capState = 0;
} else if (capState == 0) {
// normal case
header->push_back(StringUtils::tolower(*p));
_header.push_back(StringUtils::tolower(*p));
if (*p == '-') {
capState = 1;
} else if (*p == ':') {
@ -713,19 +708,19 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
}
} else {
// output as is
header->push_back(*p);
_header.push_back(*p);
}
++p;
}
header->append(": ", 2);
header->append(it.second);
header->append("\r\n", 2);
_header.append(": ", 2);
_header.append(it.second);
_header.append("\r\n", 2);
}
// add "Server" response header
if (!seenServerHeader && !HttpResponse::HIDE_PRODUCT_HEADER) {
header->append(TRI_CHAR_LENGTH_PAIR("Server: ArangoDB\r\n"));
_header.append(TRI_CHAR_LENGTH_PAIR("Server: ArangoDB\r\n"));
}
// turn on the keepAlive timer
@ -743,32 +738,32 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
s->close();
});
header->append(TRI_CHAR_LENGTH_PAIR("Connection: Keep-Alive\r\n"));
_header.append(TRI_CHAR_LENGTH_PAIR("Connection: Keep-Alive\r\n"));
} else {
header->append(TRI_CHAR_LENGTH_PAIR("Connection: Close\r\n"));
_header.append(TRI_CHAR_LENGTH_PAIR("Connection: Close\r\n"));
}
// add "Content-Type" header
switch (response.contentType()) {
case ContentType::UNSET:
case ContentType::JSON:
header->append(TRI_CHAR_LENGTH_PAIR(
_header.append(TRI_CHAR_LENGTH_PAIR(
"Content-Type: application/json; charset=utf-8\r\n"));
break;
case ContentType::VPACK:
header->append(
_header.append(
TRI_CHAR_LENGTH_PAIR("Content-Type: application/x-velocypack\r\n"));
break;
case ContentType::TEXT:
header->append(
_header.append(
TRI_CHAR_LENGTH_PAIR("Content-Type: text/plain; charset=utf-8\r\n"));
break;
case ContentType::HTML:
header->append(
_header.append(
TRI_CHAR_LENGTH_PAIR("Content-Type: text/html; charset=utf-8\r\n"));
break;
case ContentType::DUMP:
header->append(TRI_CHAR_LENGTH_PAIR(
_header.append(TRI_CHAR_LENGTH_PAIR(
"Content-Type: application/x-arango-dump; charset=utf-8\r\n"));
break;
case ContentType::CUSTOM: // don't do anything
@ -776,45 +771,61 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
}
for (auto const& it : response.cookies()) {
header->append(TRI_CHAR_LENGTH_PAIR("Set-Cookie: "));
header->append(it);
header->append("\r\n", 2);
_header.append(TRI_CHAR_LENGTH_PAIR("Set-Cookie: "));
_header.append(it);
_header.append("\r\n", 2);
}
size_t len = response.bodySize();
header->append(TRI_CHAR_LENGTH_PAIR("Content-Length: "));
header->append(std::to_string(len));
header->append("\r\n\r\n", 4);
_header.append(TRI_CHAR_LENGTH_PAIR("Content-Length: "));
_header.append(std::to_string(len));
_header.append("\r\n\r\n", 4);
std::shared_ptr<basics::StringBuffer> body = response.stealBody();
TRI_ASSERT(_response == nullptr);
_response = response.stealBody();
// append write buffer and statistics
double const totalTime = RequestStatistics::ELAPSED_SINCE_READ_START(stat);
// and give some request information
LOG_TOPIC("8f555", DEBUG, Logger::REQUESTS)
<< "\"http-request-end\",\"" << (void*)this << "\",\"" << this->_connectionInfo.clientAddress
<< "\",\"" << GeneralRequest::translateMethod(::llhttpToRequestType(&_parser)) << "\",\""
<< static_cast<int>(response.responseCode()) << "\"," << Logger::FIXED(totalTime, 6);
<< "\"http-request-end\",\"" << (void*)this << "\",\""
<< this->_connectionInfo.clientAddress << "\",\""
<< GeneralRequest::translateMethod(::llhttpToRequestType(&_parser))
<< "\",\"" << static_cast<int>(response.responseCode()) << "\","
<< Logger::FIXED(totalTime, 6);
if constexpr (SocketType::Ssl == T) {
this->_protocol->context.io_context.dispatch([self = this->shared_from_this(), stat]() mutable {
auto* thisPtr = static_cast<HttpCommTask<T>*>(self.get());
thisPtr->writeResponse(stat);
});
} else {
writeResponse(stat);
}
}
// called on IO context thread
template <SocketType T>
void HttpCommTask<T>::writeResponse(RequestStatistics* stat) {
TRI_ASSERT(!_header.empty());
RequestStatistics::SET_WRITE_START(stat);
this->_protocol->context.io_context.post([this, self = this->shared_from_this(), header = std::move(header), body = std::move(body), stat] () mutable {
std::array<asio_ns::const_buffer, 2> buffers;
buffers[0] = asio_ns::buffer(header->data(), header->size());
buffers[0] = asio_ns::buffer(_header.data(), _header.size());
if (HTTP_HEAD != _parser.method) {
buffers[1] = asio_ns::buffer(body->data(), body->size());
buffers[1] = asio_ns::buffer(_response->data(), _response->size());
}
// FIXME measure performance w/o sync write
asio_ns::async_write(this->_protocol->socket, buffers,
[self = std::move(self),
h = std::move(header),
b = std::move(body),
stat](asio_ns::error_code ec, size_t nwrite) {
[self = this->shared_from_this(), stat](asio_ns::error_code ec, size_t nwrite) {
auto* thisPtr = static_cast<HttpCommTask<T>*>(self.get());
RequestStatistics::SET_WRITE_END(stat);
RequestStatistics::ADD_SENT_BYTES(stat, h->size() + b->size());
RequestStatistics::ADD_SENT_BYTES(stat, nwrite);
thisPtr->_response.reset();
llhttp_errno_t err = llhttp_get_errno(&thisPtr->_parser);
if (ec || !thisPtr->_shouldKeepAlive || err != HPE_PAUSED) {
if (ec) {
@ -823,7 +834,6 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
}
thisPtr->close();
} else { // ec == HPE_PAUSED
llhttp_resume(&thisPtr->_parser);
thisPtr->asyncReadSome();
}
@ -831,7 +841,6 @@ void HttpCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes,
stat->release();
}
});
});
}
template <SocketType T>

10
arangod/GeneralServer/HttpCommTask.h
View File

@ -33,6 +33,10 @@
namespace arangodb {
class HttpRequest;
namespace basics {
class StringBuffer;
}
namespace rest {
template <SocketType T>
@ -85,16 +89,22 @@ private:
/// decompress content
bool handleContentEncoding(HttpRequest&);
// called on IO context thread
void writeResponse(RequestStatistics* stat);
private:
/// the node http-parser
llhttp_t _parser;
llhttp_settings_t _parserSettings;
velocypack::Buffer<uint8_t> _header;
// ==== parser state ====
std::string _lastHeaderField;
std::string _lastHeaderValue;
std::string _origin; // value of the HTTP origin header the client sent (if
std::unique_ptr<HttpRequest> _request;
std::unique_ptr<basics::StringBuffer> _response;
bool _lastHeaderWasValue;
bool _shouldKeepAlive; /// keep connection open
bool _messageDone;

9
arangod/GeneralServer/VstCommTask.cpp
View File

@ -334,9 +334,14 @@ void VstCommTask<T>::sendResponse(std::unique_ptr<GeneralResponse> baseRes, Requ
bool expected = _writing.load();
if (false == expected) {
if (_writing.compare_exchange_strong(expected, true)) {
this->_protocol->context.io_context.post([this, self = this->shared_from_this()]{
this->doWrite(); // we managed to start writing
// we managed to start writing
if constexpr (SocketType::Ssl == T) {
this->_protocol->context.io_context.post([self = this->shared_from_this()]() {
static_cast<VstCommTask<T>*>(self.get())->doWrite();
});
} else {
doWrite();
}
}
}
}

53
js/server/modules/@arangodb/cluster.js
View File

@ -89,58 +89,6 @@ var shardList = function (dbName, collectionName) {
return shards;
};
// /////////////////////////////////////////////////////////////////////////////
// / @brief wait for a distributed response
// /////////////////////////////////////////////////////////////////////////////
var waitForDistributedResponse = function (data, numberOfRequests, ignoreHttpErrors) {
var received = [];
try {
while (received.length < numberOfRequests) {
var result = global.ArangoClusterComm.wait(data);
var status = result.status;
if (status === 'ERROR') {
raiseError(arangodb.errors.ERROR_INTERNAL.code,
'received an error from a DB server: ' + JSON.stringify(result));
} else if (status === 'TIMEOUT') {
raiseError(arangodb.errors.ERROR_CLUSTER_TIMEOUT.code,
arangodb.errors.ERROR_CLUSTER_TIMEOUT.message);
} else if (status === 'DROPPED') {
raiseError(arangodb.errors.ERROR_INTERNAL.code,
'the operation was dropped');
} else if (status === 'RECEIVED') {
received.push(result);
if (result.headers && result.headers.hasOwnProperty('x-arango-response-code')) {
var code = parseInt(result.headers['x-arango-response-code'].substr(0, 3), 10);
result.statusCode = code;
if (code >= 400 && !ignoreHttpErrors) {
var body;
try {
body = JSON.parse(result.body);
} catch (err) {
raiseError(arangodb.errors.ERROR_INTERNAL.code,
'error parsing JSON received from a DB server: ' + err.message);
}
raiseError(body.errorNum,
body.errorMessage);
}
}
} else {
// something else... wait without GC
require('internal').wait(0.1, false);
}
}
} finally {
global.ArangoClusterComm.drop(data);
}
return received;
};
// /////////////////////////////////////////////////////////////////////////////
// / @brief whether or not clustering is enabled
// /////////////////////////////////////////////////////////////////////////////
@ -493,7 +441,6 @@ exports.isCoordinator = isCoordinator;
exports.role = role;
exports.shardList = shardList;
exports.status = status;
exports.wait = waitForDistributedResponse;
exports.endpointToURL = endpointToURL;
exports.shardDistribution = shardDistribution;
exports.collectionShardDistribution = collectionShardDistribution;

1
lib/Basics/StaticStrings.cpp
View File

@ -201,7 +201,6 @@ std::string const StaticStrings::PotentialDirtyRead(
"x-arango-potential-dirty-read");
std::string const StaticStrings::RequestForwardedTo(
"x-arango-request-forwarded-to");
std::string const StaticStrings::ResponseCode("x-arango-response-code");
std::string const StaticStrings::Server("server");
std::string const StaticStrings::TransferEncoding("transfer-encoding");
std::string const StaticStrings::TransactionBody("x-arango-trx-body");

1
lib/Basics/StaticStrings.h
View File

@ -186,7 +186,6 @@ class StaticStrings {
static std::string const Origin;
static std::string const PotentialDirtyRead;
static std::string const RequestForwardedTo;
static std::string const ResponseCode;
static std::string const Server;
static std::string const TransferEncoding;
static std::string const TransactionBody;

2
lib/Rest/HttpRequest.cpp
View File

@ -534,7 +534,7 @@ void HttpRequest::parseUrl(const char* path, size_t length) {
}
}
void HttpRequest::setHeaderV2(std::string key, std::string value) {
void HttpRequest::setHeaderV2(std::string&& key, std::string&& value) {
StringUtils::tolowerInPlace(key); // always lowercase key
if (key == StaticStrings::ContentLength) {

2
lib/Rest/HttpRequest.h
View File

@ -102,7 +102,7 @@ class HttpRequest final : public GeneralRequest {
/// @brief parse an existing url
void parseUrl(char const* start, size_t len);
void setHeaderV2(std::string key, std::string value);
void setHeaderV2(std::string&& key, std::string&& value);
static HttpRequest* createHttpRequest(ContentType contentType,
char const* body, int64_t contentLength,