1
0
Fork 0
arangodb/arangod/GeneralServer/VppCommTask.cpp

780 lines
24 KiB
C++

////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// vpp://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Dr. Frank Celler
/// @author Achim Brandt
////////////////////////////////////////////////////////////////////////////////
#include "VppCommTask.h"
#include "Basics/HybridLogicalClock.h"
#include "GeneralServer/GeneralServer.h"
#include "GeneralServer/GeneralServerFeature.h"
#include "GeneralServer/RestHandler.h"
#include "GeneralServer/RestHandlerFactory.h"
#include "Scheduler/Scheduler.h"
#include "Scheduler/SchedulerFeature.h"
#include "VocBase/ticks.h"
#include <velocypack/Validator.h>
#include <velocypack/velocypack-aliases.h>
#include <stdexcept>
using namespace arangodb;
using namespace arangodb::basics;
using namespace arangodb::rest;
VppCommTask::VppCommTask(GeneralServer* server, TRI_socket_t sock,
ConnectionInfo&& info, double timeout)
: Task("VppCommTask"),
GeneralCommTask(server, sock, std::move(info), timeout),
_requestType(GeneralRequest::RequestType::ILLEGAL),
_fullUrl() {
_protocol = "vpp";
// connectionStatisticsAgentSetVpp();
}
namespace {
constexpr size_t getChunkHeaderLength(bool oneChunk) {
return (oneChunk ? 2 * sizeof(uint32_t) + 1 * sizeof(uint64_t)
: 2 * sizeof(uint32_t) + 2 * sizeof(uint64_t));
}
}
void VppCommTask::addResponse(VppResponse* response, bool isError) {
/*
_requestPending = false;
_isChunked = false;
if (isError) {
resetState(true);
}
// CORS response handling
if (!_origin.empty()) {
// the request contained an Origin header. We have to send back the
// access-control-allow-origin header now
LOG(TRACE) << "handling CORS response";
response->setHeaderNC(StaticStrings::AccessControlExposeHeaders,
StaticStrings::ExposedCorsHeaders);
// send back original value of "Origin" header
response->setHeaderNC(StaticStrings::AccessControlAllowOrigin, _origin);
// send back "Access-Control-Allow-Credentials" header
response->setHeaderNC(StaticStrings::AccessControlAllowCredentials,
(_denyCredentials ? "false" : "true"));
}
// CORS request handling EOF
// set "connection" header
// keep-alive is the default
response->setConnectionType(_closeRequested
? VppResponse::CONNECTION_CLOSE
: VppResponse::CONNECTION_KEEP_ALIVE);
size_t const responseBodyLength = response->bodySize();
if (_requestType == GeneralRequest::RequestType::HEAD) {
// clear body if this is an vpp HEAD request
// HEAD must not return a body
response->headResponse(responseBodyLength);
}
// reserve a buffer with some spare capacity
auto buffer = std::make_unique<StringBuffer>(TRI_UNKNOWN_MEM_ZONE,
responseBodyLength + 128, false);
// write header
response->writeHeader(buffer.get());
// write body
if (_requestType != GeneralRequest::RequestType::HEAD) {
if (_isChunked) {
if (0 != responseBodyLength) {
buffer->appendHex(response->body().length());
buffer->appendText(TRI_CHAR_LENGTH_PAIR("\r\n"));
buffer->appendText(response->body());
buffer->appendText(TRI_CHAR_LENGTH_PAIR("\r\n"));
}
} else {
buffer->appendText(response->body());
}
}
buffer->ensureNullTerminated();
_writeBuffers.push_back(buffer.get());
auto b = buffer.release();
if (!b->empty()) {
LOG_TOPIC(TRACE, Logger::REQUESTS)
<< "\"vpp-request-response\",\"" << (void*)this << "\",\""
<< StringUtils::escapeUnicode(std::string(b->c_str(), b->length()))
<< "\"";
}
// clear body
response->body().clear();
double const totalTime = RequestStatisticsAgent::elapsedSinceReadStart();
_writeBuffersStats.push_back(RequestStatisticsAgent::steal());
LOG_TOPIC(INFO, Logger::REQUESTS)
<< "\"vpp-request-end\",\"" << (void*)this << "\",\""
<< _connectionInfo.clientAddress << "\",\""
<< VppRequest::translateMethod(_requestType) << "\",\""
<< VppRequest::translateVersion(_protocolVersion) << "\","
<< static_cast<int>(response->responseCode()) << ","
<< _originalBodyLength << "," << responseBodyLength << ",\"" << _fullUrl
<< "\"," << Logger::FIXED(totalTime, 6);
// start output
fillWriteBuffer();
*/
}
VppCommTask::ChunkHeader VppCommTask::readChunkHeader() {
VppCommTask::ChunkHeader header;
auto cursor = _readBuffer->begin();
uint32_t vpChunkLength;
std::memcpy(&header._length, cursor, sizeof(header._length));
cursor += sizeof(header._length);
uint32_t chunkX;
std::memcpy(&chunkX, cursor, sizeof(chunkX));
cursor += sizeof(chunkX);
header._isFirst = chunkX & 0x1;
header._chunk = chunkX >> 1;
std::memcpy(&header._messageId, cursor, sizeof(header._messageId));
return header;
}
bool VppCommTask::chunkComplete() {
auto start = _readBuffer->begin();
std::size_t length = std::distance(start, _readBuffer->end());
auto& prv = _processReadVariables;
if (!prv._currentChunkLength && length < sizeof(uint32_t)) {
return false;
}
if (!prv._currentChunkLength) {
// read chunk length
std::memcpy(&prv._currentChunkLength, start, sizeof(uint32_t));
}
if (length < prv._currentChunkLength) {
// chunk not complete
return false;
}
return true;
}
std::pair<std::size_t, std::size_t> VppCommTask::validateChunkOnBuffer(
std::size_t chunkLength) {
auto readBufferStart = _readBuffer->begin();
auto chunkHeaderLength = getChunkHeaderLength(true);
auto sliceStart = readBufferStart + chunkHeaderLength;
auto sliceLength = chunkLength - chunkHeaderLength;
VPackValidator validator;
// check for slice start to the end of Chunk
// isSubPart allows the slice to be shorter than the checked buffer.
validator.validate(sliceStart, sliceLength, /*isSubPart =*/true);
VPackSlice vppHeader(sliceStart);
auto vppHeaderLen = vppHeader.byteSize();
sliceStart += vppHeaderLen;
sliceLength += vppHeaderLen;
validator.validate(sliceStart, sliceLength, /*isSubPart =*/true);
VPackSlice vppPayload(sliceStart);
auto vppPayloadLen = vppPayload.byteSize();
return std::pair<std::size_t, std::size_t>(vppHeaderLen,
vppHeaderLen + vppPayloadLen);
}
// reads data from the socket
bool VppCommTask::processRead() {
auto readBufferStart = _readBuffer->begin();
if (readBufferStart == nullptr || !chunkComplete()) {
return true; // no data or incomplete
}
auto header = readChunkHeader();
if (header._isFirst && header._chunk == 1) { // one chunk one message
validateChunkOnBuffer(header._length);
VPackMessage message;
// execute
}
auto incompleteMessageItr = _incompleteMessages.find(header._messageId);
if (header._isFirst) { // first chunk of multi chunk message
if (incompleteMessageItr != _incompleteMessages.end()) {
throw std::logic_error(
"Message should be first but is already in the Map of incomplete "
"messages");
}
// append to VPackBuffer in incomplete Message
auto numberOfChunks = header._chunk;
auto chunkNumber = 1;
// set message length
} else { // followup chunk of some mesage
if (incompleteMessageItr == _incompleteMessages.end()) {
throw std::logic_error("found message without previous part");
}
auto& im = incompleteMessageItr->second; // incomplete Message
auto chunkNumber = header._chunk;
// append to VPackBuffer in incomplete Message
if (chunkNumber == im._numberOfChunks) {
// VPackMessage message;
// execute
}
}
// clean buffer up to lenght of chunk
return true;
/*
if (RequestStatisticsAgent::_statistics != nullptr) {
RequestStatisticsAgent::_statistics->_id = (void*)this;
}
#endif
_newRequest = false;
_startPosition = _readPosition;
_protocolVersion = GeneralRequest::ProtocolVersion::UNKNOWN;
_requestType = GeneralRequest::RequestType::ILLEGAL;
_fullUrl = "";
_denyCredentials = true;
_acceptDeflate = false;
_sinceCompactification++;
}
char const* end = etr - 3;
// read buffer contents are way to small. we can exit here directly
if (ptr >= end) {
return false;
}
// request started
requestStatisticsAgentSetReadStart();
// check for the end of the request
for (; ptr < end; ptr++) {
if (ptr[0] == '\r' && ptr[1] == '\n' && ptr[2] == '\r' &&
ptr[3] == '\n') {
break;
}
}
// check if header is too large
size_t headerLength = ptr - (_readBuffer->c_str() + _startPosition);
if (headerLength > MaximalHeaderSize) {
LOG(WARN) << "maximal header size is " << MaximalHeaderSize
<< ", request header size is " << headerLength;
// header is too large
handleSimpleError(
GeneralResponse::ResponseCode::REQUEST_HEADER_FIELDS_TOO_LARGE);
return false;
}
// header is complete
if (ptr < end) {
_readPosition = ptr - _readBuffer->c_str() + 4;
LOG(TRACE) << "vpp READ FOR " << (void*)this << ": "
<< std::string(_readBuffer->c_str() + _startPosition,
_readPosition - _startPosition);
// check that we know, how to serve this request and update the
connection
// information, i. e. client and server addresses and ports and create a
// request context for that request
_request = new VppRequest(
_connectionInfo, _readBuffer->c_str() + _startPosition,
_readPosition - _startPosition, _allowMethodOverride);
GeneralServerFeature::HANDLER_FACTORY->setRequestContext(_request);
_request->setClientTaskId(_taskId);
// check vpp protocol version
_protocolVersion = _request->protocolVersion();
if (_protocolVersion != GeneralRequest::ProtocolVersion::vpp_1_0 &&
_protocolVersion != GeneralRequest::ProtocolVersion::vpp_1_1) {
handleSimpleError(
GeneralResponse::ResponseCode::vpp_VERSION_NOT_SUPPORTED);
return false;
}
// check max URL length
_fullUrl = _request->fullUrl();
if (_fullUrl.size() > 16384) {
handleSimpleError(GeneralResponse::ResponseCode::REQUEST_URI_TOO_LONG);
return false;
}
// update the connection information, i. e. client and server addresses
// and ports
_request->setProtocol(_protocol);
LOG(TRACE) << "server port " << _connectionInfo.serverPort
<< ", client port " << _connectionInfo.clientPort;
// set body start to current position
_bodyPosition = _readPosition;
_bodyLength = 0;
// keep track of the original value of the "origin" request header (if
// any), we need this value to handle CORS requests
_origin = _request->header(StaticStrings::Origin);
if (!_origin.empty()) {
// check for Access-Control-Allow-Credentials header
bool found;
std::string const& allowCredentials = _request->header(
StaticStrings::AccessControlAllowCredentials, found);
if (found) {
// default is to allow nothing
_denyCredentials = true;
// if the request asks to allow credentials, we'll check against the
// configured whitelist of origins
std::vector<std::string> const& accessControlAllowOrigins =
GeneralServerFeature::accessControlAllowOrigins();
if (StringUtils::boolean(allowCredentials) &&
!accessControlAllowOrigins.empty()) {
if (accessControlAllowOrigins[0] == "*") {
// special case: allow everything
_denyCredentials = false;
} else if (!_origin.empty()) {
// copy origin string
if (_origin[_origin.size() - 1] == '/') {
// strip trailing slash
auto result = std::find(accessControlAllowOrigins.begin(),
accessControlAllowOrigins.end(),
_origin.substr(0, _origin.size() -
1));
_denyCredentials = (result ==
accessControlAllowOrigins.end());
} else {
auto result =
std::find(accessControlAllowOrigins.begin(),
accessControlAllowOrigins.end(), _origin);
_denyCredentials = (result ==
accessControlAllowOrigins.end());
}
} else {
TRI_ASSERT(_denyCredentials);
}
}
}
}
// store the original request's type. we need it later when responding
// (original request object gets deleted before responding)
_requestType = _request->requestType();
requestStatisticsAgentSetRequestType(_requestType);
// handle different vpp methods
switch (_requestType) {
case GeneralRequest::RequestType::GET:
case GeneralRequest::RequestType::DELETE_REQ:
case GeneralRequest::RequestType::HEAD:
case GeneralRequest::RequestType::OPTIONS:
case GeneralRequest::RequestType::POST:
case GeneralRequest::RequestType::PUT:
case GeneralRequest::RequestType::PATCH: {
// technically, sending a body for an vpp DELETE request is not
// forbidden, but it is not explicitly supported
bool const expectContentLength =
(_requestType == GeneralRequest::RequestType::POST ||
_requestType == GeneralRequest::RequestType::PUT ||
_requestType == GeneralRequest::RequestType::PATCH ||
_requestType == GeneralRequest::RequestType::OPTIONS ||
_requestType == GeneralRequest::RequestType::DELETE_REQ);
if (!checkContentLength(expectContentLength)) {
return false;
}
if (_bodyLength == 0) {
handleRequest = true;
}
break;
}
default: {
size_t l = _readPosition - _startPosition;
if (6 < l) {
l = 6;
}
LOG(WARN) << "got corrupted vpp request '"
<< std::string(_readBuffer->c_str() + _startPosition, l)
<< "'";
// force a socket close, response will be ignored!
TRI_CLOSE_SOCKET(_commSocket);
TRI_invalidatesocket(&_commSocket);
// bad request, method not allowed
handleSimpleError(GeneralResponse::ResponseCode::METHOD_NOT_ALLOWED);
return false;
}
}
//
.............................................................................
// check if server is active
//
.............................................................................
Scheduler const* scheduler = SchedulerFeature::SCHEDULER;
if (scheduler != nullptr && !scheduler->isActive()) {
// server is inactive and will intentionally respond with vpp 503
LOG(TRACE) << "cannot serve request - server is inactive";
handleSimpleError(GeneralResponse::ResponseCode::SERVICE_UNAVAILABLE);
return false;
}
// check for a 100-continue
if (_readRequestBody) {
bool found;
std::string const& expect =
_request->header(StaticStrings::Expect, found);
if (found && StringUtils::trim(expect) == "100-continue") {
LOG(TRACE) << "received a 100-continue request";
auto buffer = std::make_unique<StringBuffer>(TRI_UNKNOWN_MEM_ZONE);
buffer->appendText(
TRI_CHAR_LENGTH_PAIR("vpp/1.1 100 (Continue)\r\n\r\n"));
buffer->ensureNullTerminated();
_writeBuffers.push_back(buffer.get());
buffer.release();
_writeBuffersStats.push_back(nullptr);
fillWriteBuffer();
}
}
} else {
size_t l = (_readBuffer->end() - _readBuffer->c_str());
if (_startPosition + 4 <= l) {
_readPosition = l - 4;
}
}
}
// readRequestBody might have changed, so cannot use else
if (_readRequestBody) {
if (_readBuffer->length() - _bodyPosition < _bodyLength) {
setKeepAliveTimeout(_keepAliveTimeout);
// let client send more
return false;
}
// read "bodyLength" from read buffer and add this body to "vppRequest"
requestAsVpp()->setBody(_readBuffer->c_str() + _bodyPosition,
_bodyLength);
LOG(TRACE) << "" << std::string(_readBuffer->c_str() + _bodyPosition,
_bodyLength);
// remove body from read buffer and reset read position
_readRequestBody = false;
handleRequest = true;
}
//
.............................................................................
// request complete
//
// we have to delete request in here or pass it to a handler, which will
// delete
// it
//
.............................................................................
if (!handleRequest) {
return false;
}
requestStatisticsAgentSetReadEnd();
requestStatisticsAgentAddReceivedBytes(_bodyPosition - _startPosition +
_bodyLength);
bool const isOptionsRequest =
(_requestType == GeneralRequest::RequestType::OPTIONS);
resetState(false);
//
.............................................................................
// keep-alive handling
//
.............................................................................
std::string connectionType =
StringUtils::tolower(_request->header(StaticStrings::Connection));
if (connectionType == "close") {
// client has sent an explicit "Connection: Close" header. we should close
// the connection
LOG(DEBUG) << "connection close requested by client";
_closeRequested = true;
} else if (requestAsVpp()->isVpp10() && connectionType != "keep-alive") {
// vpp 1.0 request, and no "Connection: Keep-Alive" header sent
// we should close the connection
LOG(DEBUG) << "no keep-alive, connection close requested by client";
_closeRequested = true;
} else if (_keepAliveTimeout <= 0.0) {
// if keepAliveTimeout was set to 0.0, we'll close even keep-alive
// connections immediately
LOG(DEBUG) << "keep-alive disabled by admin";
_closeRequested = true;
}
// we keep the connection open in all other cases (vpp 1.1 or Keep-Alive
// header sent)
//
.............................................................................
// authenticate
//
.............................................................................
GeneralResponse::ResponseCode authResult = authenticateRequest();
// authenticated or an OPTIONS request. OPTIONS requests currently go
// unauthenticated
if (authResult == GeneralResponse::ResponseCode::OK || isOptionsRequest) {
// handle vpp OPTIONS requests directly
if (isOptionsRequest) {
processCorsOptions();
} else {
processRequest();
}
}
// not found
else if (authResult == GeneralResponse::ResponseCode::NOT_FOUND) {
handleSimpleError(authResult, TRI_ERROR_ARANGO_DATABASE_NOT_FOUND,
TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND));
}
// forbidden
else if (authResult == GeneralResponse::ResponseCode::FORBIDDEN) {
handleSimpleError(authResult, TRI_ERROR_USER_CHANGE_PASSWORD,
"change password");
}
// not authenticated
else {
VppResponse response(GeneralResponse::ResponseCode::UNAUTHORIZED);
std::string realm = "Bearer token_type=\"JWT\", realm=\"ArangoDB\"";
response.setHeaderNC(StaticStrings::WwwAuthenticate, std::move(realm));
clearRequest();
processResponse(&response);
}
*/
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief processes a request
////////////////////////////////////////////////////////////////////////////////
void VppCommTask::processRequest() {
/*
// check for deflate
bool found;
auto vppRequest = requestAsVpp();
std::string const& acceptEncoding =
vppRequest->header(StaticStrings::AcceptEncoding, found);
if (found) {
if (acceptEncoding.find("deflate") != std::string::npos) {
_acceptDeflate = true;
}
}
if (vppRequest != nullptr) {
LOG_TOPIC(DEBUG, Logger::REQUESTS)
<< "\"vpp-request-begin\",\"" << (void*)this << "\",\""
<< _connectionInfo.clientAddress << "\",\""
<< VppRequest::translateMethod(_requestType) << "\",\""
<< VppRequest::translateVersion(_protocolVersion) << "\"," << _fullUrl
<< "\"";
std::string const& body = vppRequest->body();
if (!body.empty()) {
LOG_TOPIC(DEBUG, Logger::REQUESTS)
<< "\"vpp-request-body\",\"" << (void*)this << "\",\""
<< (StringUtils::escapeUnicode(body)) << "\"";
}
}
// check for an HLC time stamp
std::string const& timeStamp =
_request->header(StaticStrings::HLCHeader, found);
if (found) {
uint64_t timeStampInt =
arangodb::basics::HybridLogicalClock::decodeTimeStampWithCheck(
timeStamp);
if (timeStampInt != 0) {
TRI_HybridLogicalClock(timeStampInt);
}
}
// create a handler and execute
executeRequest(_request,
new
VppResponse(GeneralResponse::ResponseCode::SERVER_ERROR));
*/
}
void VppCommTask::completedWriteBuffer() {
// _writeBuffer = nullptr;
// _writeLength = 0;
//
// if (_writeBufferStatistics != nullptr) {
// _writeBufferStatistics->_writeEnd = TRI_StatisticsTime();
// TRI_ReleaseRequestStatistics(_writeBufferStatistics);
// _writeBufferStatistics = nullptr;
// }
//
// fillWriteBuffer();
//
// if (!_clientClosed && _closeRequested && !hasWriteBuffer() &&
// _writeBuffers.empty() && !_isChunked) {
// _clientClosed = true;
// }
}
void VppCommTask::resetState(bool close) {
/*
if (close) {
clearRequest();
_requestPending = false;
_isChunked = false;
_closeRequested = true;
_readPosition = 0;
_bodyPosition = 0;
_bodyLength = 0;
} else {
_requestPending = true;
bool compact = false;
if (_sinceCompactification > RunCompactEvery) {
compact = true;
} else if (_readBuffer->length() > MaximalPipelineSize) {
compact = true;
}
if (compact) {
_readBuffer->erase_front(_bodyPosition + _bodyLength);
_sinceCompactification = 0;
_readPosition = 0;
} else {
_readPosition = _bodyPosition + _bodyLength;
if (_readPosition == _readBuffer->length()) {
_sinceCompactification = 0;
_readPosition = 0;
_readBuffer->reset();
}
}
_bodyPosition = 0;
_bodyLength = 0;
}
_newRequest = true;
_readRequestBody = false;
*/
}
// GeneralResponse::ResponseCode VppCommTask::authenticateRequest() {
// auto context = (_request == nullptr) ? nullptr :
// _request->requestContext();
//
// if (context == nullptr && _request != nullptr) {
// bool res =
// GeneralServerFeature::HANDLER_FACTORY->setRequestContext(_request);
//
// if (!res) {
// return GeneralResponse::ResponseCode::NOT_FOUND;
// }
//
// context = _request->requestContext();
// }
//
// if (context == nullptr) {
// return GeneralResponse::ResponseCode::SERVER_ERROR;
// }
//
// return context->authenticate();
// }
// convert internal GeneralRequest to VppRequest
VppRequest* VppCommTask::requestAsVpp() {
VppRequest* request = dynamic_cast<VppRequest*>(_request);
if (request == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
}
return request;
};