1169 lines
48 KiB
C++
1169 lines
48 KiB
C++
// Protocol Buffers - Google's data interchange format
|
|
// Copyright 2008 Google Inc. All rights reserved.
|
|
//
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file or at
|
|
// https://developers.google.com/open-source/licenses/bsd
|
|
|
|
#ifndef GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__
|
|
#define GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__
|
|
|
|
#include <cassert>
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <string>
|
|
#include <type_traits>
|
|
|
|
#include "absl/base/optimization.h"
|
|
#include "absl/log/absl_log.h"
|
|
#include "absl/strings/cord.h"
|
|
#include "absl/strings/string_view.h"
|
|
#include "google/protobuf/extension_set.h"
|
|
#include "google/protobuf/generated_message_tctable_decl.h"
|
|
#include "google/protobuf/map.h"
|
|
#include "google/protobuf/message_lite.h"
|
|
#include "google/protobuf/metadata_lite.h"
|
|
#include "google/protobuf/parse_context.h"
|
|
#include "google/protobuf/port.h"
|
|
#include "google/protobuf/raw_ptr.h"
|
|
#include "google/protobuf/repeated_field.h"
|
|
#include "google/protobuf/repeated_ptr_field.h"
|
|
#include "google/protobuf/serial_arena.h"
|
|
#include "google/protobuf/wire_format_lite.h"
|
|
|
|
// Must come last:
|
|
#include "google/protobuf/port_def.inc"
|
|
|
|
namespace google {
|
|
namespace protobuf {
|
|
|
|
class Message;
|
|
class UnknownFieldSet;
|
|
|
|
namespace internal {
|
|
|
|
enum {
|
|
kInlinedStringAuxIdx = 0,
|
|
kSplitOffsetAuxIdx = 1,
|
|
kSplitSizeAuxIdx = 2,
|
|
};
|
|
|
|
// Field layout enums.
|
|
//
|
|
// Structural information about fields is packed into a 16-bit value. The enum
|
|
// types below represent bitwise fields, along with their respective widths,
|
|
// shifts, and masks.
|
|
//
|
|
// Bit:
|
|
// +-----------------------+-----------------------+
|
|
// |15 .. 8|7 .. 0|
|
|
// +-----------------------+-----------------------+
|
|
// : . : . : . : . : . : . : 3|========| [3] FieldType
|
|
// : : : : : : . 4|==| : : [1] FieldSplit
|
|
// : : : : : 6|=====| . : : [2] FieldCardinality
|
|
// : . : . : . : 9|========| . : . : . : [3] FieldRep
|
|
// : : :11|=====| : : : : : [2] TransformValidation
|
|
// : . :13|=====| : . : . : . : . : . : [2] FormatDiscriminator
|
|
// +-----------------------+-----------------------+
|
|
// |15 .. 8|7 .. 0|
|
|
// +-----------------------+-----------------------+
|
|
//
|
|
namespace field_layout {
|
|
// clang-format off
|
|
|
|
|
|
// Field kind (3 bits):
|
|
// These values broadly represent a wire type and an in-memory storage class.
|
|
enum FieldKind : uint16_t {
|
|
kFkShift = 0,
|
|
kFkBits = 3,
|
|
kFkMask = ((1 << kFkBits) - 1) << kFkShift,
|
|
|
|
kFkNone = 0,
|
|
kFkVarint, // WT=0 rep=8,32,64 bits
|
|
kFkPackedVarint, // WT=2 rep=8,32,64 bits
|
|
kFkFixed, // WT=1,5 rep=32,64 bits
|
|
kFkPackedFixed, // WT=2 rep=32,64 bits
|
|
kFkString, // WT=2 rep=various
|
|
kFkMessage, // WT=2,3,4 rep=MessageLite*
|
|
// Maps are a special case of Message, but use different parsing logic.
|
|
kFkMap, // WT=2 rep=Map(Lite)<various, various>
|
|
};
|
|
|
|
static_assert(kFkMap < (1 << kFkBits), "too many types");
|
|
|
|
// Split (1 bit):
|
|
enum FieldSplit : uint16_t {
|
|
kSplitShift = kFkShift+ kFkBits,
|
|
kSplitBits = 1,
|
|
kSplitMask = ((1 << kSplitBits) - 1) << kSplitShift,
|
|
|
|
kSplitFalse = 0,
|
|
kSplitTrue = 1 << kSplitShift,
|
|
};
|
|
|
|
// Cardinality (2 bits):
|
|
// These values determine how many values a field can have and its presence.
|
|
// Packed fields are represented in FieldType.
|
|
enum Cardinality : uint16_t {
|
|
kFcShift = kSplitShift+ kSplitBits,
|
|
kFcBits = 2,
|
|
kFcMask = ((1 << kFcBits) - 1) << kFcShift,
|
|
|
|
kFcSingular = 0,
|
|
kFcOptional = 1 << kFcShift,
|
|
kFcRepeated = 2 << kFcShift,
|
|
kFcOneof = 3 << kFcShift,
|
|
};
|
|
|
|
|
|
// Field representation (3 bits):
|
|
// These values are the specific refinements of storage classes in FieldType.
|
|
enum FieldRep : uint16_t {
|
|
kRepShift = kFcShift + kFcBits,
|
|
kRepBits = 3,
|
|
kRepMask = ((1 << kRepBits) - 1) << kRepShift,
|
|
|
|
// Numeric types (used for optional and repeated fields):
|
|
kRep8Bits = 0,
|
|
kRep32Bits = 2 << kRepShift,
|
|
kRep64Bits = 3 << kRepShift,
|
|
// String types:
|
|
kRepAString = 0, // ArenaStringPtr
|
|
kRepIString = 1 << kRepShift, // InlinedString
|
|
kRepCord = 2 << kRepShift, // absl::Cord
|
|
kRepSPiece = 3 << kRepShift, // StringPieceField
|
|
kRepSString = 4 << kRepShift, // std::string*
|
|
// Message types (WT=2 unless otherwise noted):
|
|
kRepMessage = 0, // MessageLite*
|
|
kRepGroup = 1 << kRepShift, // MessageLite* (WT=3,4)
|
|
kRepLazy = 2 << kRepShift, // LazyField*
|
|
};
|
|
|
|
// Transform/validation (2 bits):
|
|
// These values determine transforms or validation to/from wire format.
|
|
enum TransformValidation : uint16_t {
|
|
kTvShift = kRepShift + kRepBits,
|
|
kTvBits = 2,
|
|
kTvMask = ((1 << kTvBits) - 1) << kTvShift,
|
|
|
|
// Varint fields:
|
|
kTvZigZag = 1 << kTvShift,
|
|
kTvEnum = 2 << kTvShift, // validate using ValidateEnum()
|
|
kTvRange = 3 << kTvShift, // validate using FieldAux::enum_range
|
|
// String fields:
|
|
kTvUtf8Debug = 1 << kTvShift, // proto2
|
|
kTvUtf8 = 2 << kTvShift, // proto3
|
|
|
|
// Message fields:
|
|
kTvDefault = 1 << kTvShift, // Aux has default_instance*
|
|
kTvTable = 2 << kTvShift, // Aux has TcParseTableBase*
|
|
kTvWeakPtr = 3 << kTvShift, // Aux has default_instance** (for weak)
|
|
|
|
// Lazy message fields:
|
|
kTvEager = 1 << kTvShift,
|
|
kTvLazy = 2 << kTvShift,
|
|
};
|
|
|
|
static_assert((kTvEnum & kTvRange) != 0,
|
|
"enum validation types must share a bit");
|
|
static_assert((kTvEnum & kTvRange & kTvZigZag) == 0,
|
|
"zigzag encoding is not enum validation");
|
|
|
|
// Format discriminators (2 bits):
|
|
enum FormatDiscriminator : uint16_t {
|
|
kFmtShift = kTvShift + kTvBits,
|
|
kFmtBits = 2,
|
|
kFmtMask = ((1 << kFmtBits) - 1) << kFmtShift,
|
|
|
|
// Numeric:
|
|
kFmtUnsigned = 1 << kFmtShift, // fixed, varint
|
|
kFmtSigned = 2 << kFmtShift, // fixed, varint
|
|
kFmtFloating = 3 << kFmtShift, // fixed
|
|
kFmtEnum = 3 << kFmtShift, // varint
|
|
// Strings:
|
|
kFmtUtf8 = 1 << kFmtShift, // string (proto3, enforce_utf8=true)
|
|
kFmtUtf8Escape = 2 << kFmtShift, // string (proto2, enforce_utf8=false)
|
|
// Bytes:
|
|
kFmtArray = 1 << kFmtShift, // bytes
|
|
// Messages:
|
|
kFmtShow = 1 << kFmtShift, // message, map
|
|
};
|
|
|
|
// Update this assertion (and comments above) when adding or removing bits:
|
|
static_assert(kFmtShift + kFmtBits == 13, "number of bits changed");
|
|
|
|
// This assertion should not change unless the storage width changes:
|
|
static_assert(kFmtShift + kFmtBits <= 16, "too many bits");
|
|
|
|
// Convenience aliases (16 bits, with format):
|
|
enum FieldType : uint16_t {
|
|
// Numeric types:
|
|
kBool = 0 | kFkVarint | kRep8Bits,
|
|
|
|
kFixed32 = 0 | kFkFixed | kRep32Bits | kFmtUnsigned,
|
|
kUInt32 = 0 | kFkVarint | kRep32Bits | kFmtUnsigned,
|
|
kSFixed32 = 0 | kFkFixed | kRep32Bits | kFmtSigned,
|
|
kInt32 = 0 | kFkVarint | kRep32Bits | kFmtSigned,
|
|
kSInt32 = 0 | kFkVarint | kRep32Bits | kFmtSigned | kTvZigZag,
|
|
kFloat = 0 | kFkFixed | kRep32Bits | kFmtFloating,
|
|
kEnum = 0 | kFkVarint | kRep32Bits | kFmtEnum | kTvEnum,
|
|
kEnumRange = 0 | kFkVarint | kRep32Bits | kFmtEnum | kTvRange,
|
|
kOpenEnum = 0 | kFkVarint | kRep32Bits | kFmtEnum,
|
|
|
|
kFixed64 = 0 | kFkFixed | kRep64Bits | kFmtUnsigned,
|
|
kUInt64 = 0 | kFkVarint | kRep64Bits | kFmtUnsigned,
|
|
kSFixed64 = 0 | kFkFixed | kRep64Bits | kFmtSigned,
|
|
kInt64 = 0 | kFkVarint | kRep64Bits | kFmtSigned,
|
|
kSInt64 = 0 | kFkVarint | kRep64Bits | kFmtSigned | kTvZigZag,
|
|
kDouble = 0 | kFkFixed | kRep64Bits | kFmtFloating,
|
|
|
|
kPackedBool = 0 | kFkPackedVarint | kRep8Bits,
|
|
|
|
kPackedFixed32 = 0 | kFkPackedFixed | kRep32Bits | kFmtUnsigned,
|
|
kPackedUInt32 = 0 | kFkPackedVarint | kRep32Bits | kFmtUnsigned,
|
|
kPackedSFixed32 = 0 | kFkPackedFixed | kRep32Bits | kFmtSigned,
|
|
kPackedInt32 = 0 | kFkPackedVarint | kRep32Bits | kFmtSigned,
|
|
kPackedSInt32 = 0 | kFkPackedVarint | kRep32Bits | kFmtSigned | kTvZigZag,
|
|
kPackedFloat = 0 | kFkPackedFixed | kRep32Bits | kFmtFloating,
|
|
kPackedEnum = 0 | kFkPackedVarint | kRep32Bits | kFmtEnum | kTvEnum,
|
|
kPackedEnumRange = 0 | kFkPackedVarint | kRep32Bits | kFmtEnum | kTvRange,
|
|
kPackedOpenEnum = 0 | kFkPackedVarint | kRep32Bits | kFmtEnum,
|
|
|
|
kPackedFixed64 = 0 | kFkPackedFixed | kRep64Bits | kFmtUnsigned,
|
|
kPackedUInt64 = 0 | kFkPackedVarint | kRep64Bits | kFmtUnsigned,
|
|
kPackedSFixed64 = 0 | kFkPackedFixed | kRep64Bits | kFmtSigned,
|
|
kPackedInt64 = 0 | kFkPackedVarint | kRep64Bits | kFmtSigned,
|
|
kPackedSInt64 = 0 | kFkPackedVarint | kRep64Bits | kFmtSigned | kTvZigZag,
|
|
kPackedDouble = 0 | kFkPackedFixed | kRep64Bits | kFmtFloating,
|
|
|
|
// String types:
|
|
kBytes = 0 | kFkString | kFmtArray,
|
|
kRawString = 0 | kFkString | kFmtUtf8 | kTvUtf8Debug,
|
|
kUtf8String = 0 | kFkString | kFmtUtf8 | kTvUtf8,
|
|
|
|
// Message types:
|
|
kMessage = kFkMessage,
|
|
|
|
// Map types:
|
|
kMap = kFkMap,
|
|
};
|
|
// clang-format on
|
|
} // namespace field_layout
|
|
|
|
#ifndef NDEBUG
|
|
PROTOBUF_EXPORT void AlignFail(std::integral_constant<size_t, 4>,
|
|
std::uintptr_t address);
|
|
PROTOBUF_EXPORT void AlignFail(std::integral_constant<size_t, 8>,
|
|
std::uintptr_t address);
|
|
inline void AlignFail(std::integral_constant<size_t, 1>,
|
|
std::uintptr_t address) {}
|
|
#endif
|
|
|
|
#define PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(fn) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(fn##S1) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(fn##S2)
|
|
|
|
#define PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(fn) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(fn) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(fn##R1) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(fn##R2)
|
|
|
|
#define PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(fn) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(fn) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(fn##P1) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(fn##P2)
|
|
|
|
#define PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(FastEndG1) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(FastEndG2)
|
|
|
|
// TcParseFunction defines the set of table driven, tail call optimized parse
|
|
// functions. This list currently does not include all types such as maps.
|
|
//
|
|
// This table identifies the logical set of functions, it does not imply that
|
|
// functions of the same name do exist, and some entries may point to thunks or
|
|
// generic implementations accepting multiple types of input.
|
|
//
|
|
// The names are encoded as follows:
|
|
// kFast<type>[<validation>][cardinality][tag_width]
|
|
//
|
|
// type:
|
|
// V8 - bool
|
|
// V32 - int32/uint32 varint
|
|
// Z32 - int32/uint32 varint with zigzag encoding
|
|
// V64 - int64/uint64 varint
|
|
// Z64 - int64/uint64 varint with zigzag encoding
|
|
// F32 - int32/uint32/float fixed width value
|
|
// F64 - int64/uint64/double fixed width value
|
|
// E - enum
|
|
// B - string (bytes)*
|
|
// S - utf8 string, verified in debug mode only*
|
|
// U - utf8 string, strictly verified*
|
|
// Gd - group
|
|
// Gt - group width table driven parse tables
|
|
// Md - message
|
|
// Mt - message width table driven parse tables
|
|
// End - End group tag
|
|
//
|
|
// * string types can have a `c` or `i` suffix, indicating the
|
|
// underlying storage type to be cord or inlined respectively.
|
|
//
|
|
// validation:
|
|
// For enums:
|
|
// v - verify
|
|
// r - verify; enum values are a contiguous range
|
|
// r0 - verify; enum values are a small contiguous range starting at 0
|
|
// r1 - verify; enum values are a small contiguous range starting at 1
|
|
// For strings:
|
|
// u - validate utf8 encoding
|
|
// v - validate utf8 encoding for debug only
|
|
//
|
|
// cardinality:
|
|
// S - singular / optional
|
|
// R - repeated
|
|
// P - packed
|
|
// G - group terminated
|
|
//
|
|
// tag_width:
|
|
// 1: single byte encoded tag
|
|
// 2: two byte encoded tag
|
|
//
|
|
// Examples:
|
|
// FastV8S1, FastZ64S2, FastEr1P2, FastBcS1, FastMtR2, FastEndG1
|
|
//
|
|
#define PROTOBUF_TC_PARSE_FUNCTION_LIST \
|
|
/* These functions have the Fast entry ABI */ \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastV8) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastV32) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastV64) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastZ32) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastZ64) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastF32) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastF64) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEv) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEr) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEr0) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_PACKED(FastEr1) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastB) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastS) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastU) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBi) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSi) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUi) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBc) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSc) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUc) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastMl) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(MessageSetWireFormatParseLoopLite) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(MessageSetWireFormatParseLoop) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(ReflectionParseLoop) \
|
|
/* These functions have the fallback ABI */ \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(GenericFallback) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(GenericFallbackLite) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(ReflectionFallback) \
|
|
PROTOBUF_TC_PARSE_FUNCTION_X(DiscardEverythingFallback)
|
|
|
|
#define PROTOBUF_TC_PARSE_FUNCTION_X(value) k##value,
|
|
enum class TcParseFunction : uint8_t { kNone, PROTOBUF_TC_PARSE_FUNCTION_LIST };
|
|
#undef PROTOBUF_TC_PARSE_FUNCTION_X
|
|
|
|
// TcParser implements most of the parsing logic for tailcall tables.
|
|
class PROTOBUF_EXPORT TcParser final {
|
|
public:
|
|
template <typename T>
|
|
static constexpr auto GetTable() -> decltype(&T::_table_.header) {
|
|
return &T::_table_.header;
|
|
}
|
|
|
|
static PROTOBUF_ALWAYS_INLINE const char* ParseMessage(
|
|
MessageLite* msg, const char* ptr, ParseContext* ctx,
|
|
const TcParseTableBase* tc_table) {
|
|
return ctx->ParseLengthDelimitedInlined(ptr, [&](const char* ptr) {
|
|
return ParseLoop(msg, ptr, ctx, tc_table);
|
|
});
|
|
}
|
|
|
|
static PROTOBUF_ALWAYS_INLINE const char* ParseGroup(
|
|
MessageLite* msg, const char* ptr, ParseContext* ctx,
|
|
const TcParseTableBase* tc_table, uint32_t start_tag) {
|
|
return ctx->ParseGroupInlined(ptr, start_tag, [&](const char* ptr) {
|
|
return ParseLoop(msg, ptr, ctx, tc_table);
|
|
});
|
|
}
|
|
|
|
// == ABI of the tail call functions ==
|
|
// All the tail call functions have the same signature as required by clang's
|
|
// `musttail` attribute. However, their ABIs are different.
|
|
// See TcFieldData's comments for details on the layouts.
|
|
// The ABIs are as follow:
|
|
//
|
|
// - The following functions ignore `data`:
|
|
// ToTagDispatch, TagDispatch, MiniParse, ToParseLoop, Error,
|
|
// FastUnknownEnumFallback.
|
|
// - FastXXX functions expect `data` with a fast table entry ABI.
|
|
// - FastEndGX functions expect `data` with a non-field entry ABI.
|
|
// - MpXXX functions expect `data` with a mini table ABI.
|
|
// - The fallback functions (both GenericFallbackXXX and the codegen ones)
|
|
// expect only the tag in `data`. In addition, if a null `ptr` is passed,
|
|
// the function is used as a way to get a UnknownFieldOps vtable, returned
|
|
// via the `const char*` return type. See `GetUnknownFieldOps()`
|
|
|
|
PROTOBUF_CC static const char* GenericFallback(PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_CC static const char* GenericFallbackLite(PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_CC static const char* ReflectionFallback(PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_CC static const char* ReflectionParseLoop(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// This fallback will discard any field that reaches there.
|
|
// Note that fields parsed via fast/MiniParse are not going to be discarded
|
|
// even when this is enabled.
|
|
PROTOBUF_CC static const char* DiscardEverythingFallback(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// These follow the "fast" function ABI but implement the whole loop for
|
|
// message_set_wire_format types.
|
|
PROTOBUF_CC static const char* MessageSetWireFormatParseLoop(
|
|
PROTOBUF_TC_PARAM_NO_DATA_DECL);
|
|
PROTOBUF_CC static const char* MessageSetWireFormatParseLoopLite(
|
|
PROTOBUF_TC_PARAM_NO_DATA_DECL);
|
|
|
|
static const char* ParseLoop(MessageLite* msg, const char* ptr,
|
|
ParseContext* ctx,
|
|
const TcParseTableBase* table);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* ParseLoopPreserveNone(
|
|
MessageLite* msg, const char* ptr, ParseContext* ctx,
|
|
const TcParseTableBase* table);
|
|
|
|
// Functions referenced by generated fast tables (numeric types):
|
|
// F: fixed V: varint Z: zigzag
|
|
// 8/32/64: storage type width (bits)
|
|
// S: singular R: repeated P: packed
|
|
// 1/2: tag length (bytes)
|
|
|
|
// Fixed:
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF32S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF32S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF32R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF32R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF32P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF32P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF64S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF64S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF64R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF64R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF64P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastF64P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Varint:
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV8S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV8S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV8R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV8R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV8P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV8P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV32S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV32S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV32R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV32R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV32P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV32P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV64S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV64S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV64R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV64R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV64P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastV64P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Varint (with zigzag):
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ32S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ32S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ32R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ32R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ32P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ32P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ64S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ64S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ64R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ64R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ64P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastZ64P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
template <typename FieldType, int unused_data_offset, int unused_hasbit_idx>
|
|
static constexpr TailCallParseFunc SingularVarintNoZag1() {
|
|
if (sizeof(FieldType) == 1) {
|
|
return &FastV8S1;
|
|
}
|
|
if (sizeof(FieldType) == 4) {
|
|
return &FastV32S1;
|
|
}
|
|
if (sizeof(FieldType) == 8) {
|
|
return &FastV64S1;
|
|
}
|
|
static_assert(sizeof(FieldType) == 1 || sizeof(FieldType) == 4 ||
|
|
sizeof(FieldType) == 8,
|
|
"");
|
|
ABSL_LOG(FATAL) << "This should be unreachable";
|
|
}
|
|
|
|
// Functions referenced by generated fast tables (closed enum):
|
|
// E: closed enum (N.B.: open enums use V32, above)
|
|
// r: enum range v: enum validator (ValidateEnum function)
|
|
// S: singular R: repeated P: packed
|
|
// 1/2: tag length (bytes)
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastErS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastErS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastErR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastErR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastErP1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastErP2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEvS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEvS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEvR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEvR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEvP1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEvP2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr0S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr0S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr0R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr0R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr0P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr0P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr1S1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr1S2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr1R1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr1R2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr1P1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEr1P2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Functions referenced by generated fast tables (string types):
|
|
// B: bytes S: string U: UTF-8 string
|
|
// (empty): ArenaStringPtr i: InlinedString
|
|
// S: singular R: repeated
|
|
// 1/2: tag length (bytes)
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastSS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastSS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastSR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastSR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBiS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBiS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastSiS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastSiS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUiS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUiS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBcS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastBcS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastScS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastScS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUcS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUcS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Functions referenced by generated fast tables (message types):
|
|
// M: message G: group
|
|
// d: default* t: TcParseTable* (the contents of aux) l: lazy
|
|
// S: singular R: repeated
|
|
// 1/2: tag length (bytes)
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMdS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMdS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGdS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGdS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMtS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMtS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGtS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGtS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMdR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMdR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGdR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGdR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMtR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMtR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGtR1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastGtR2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMlS1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastMlS2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// NOTE: Do not dedup RefAt by having one call the other with a const_cast. It
|
|
// causes ICEs of gcc 7.5.
|
|
// https://github.com/protocolbuffers/protobuf/issues/13715
|
|
template <typename T>
|
|
static inline T& RefAt(void* x, size_t offset) {
|
|
T* target = reinterpret_cast<T*>(static_cast<char*>(x) + offset);
|
|
#if !defined(NDEBUG) && !(defined(_MSC_VER) && defined(_M_IX86))
|
|
// Check the alignment in debug mode, except in 32-bit msvc because it does
|
|
// not respect the alignment as expressed by `alignof(T)`
|
|
if (PROTOBUF_PREDICT_FALSE(
|
|
reinterpret_cast<uintptr_t>(target) % alignof(T) != 0)) {
|
|
AlignFail(std::integral_constant<size_t, alignof(T)>(),
|
|
reinterpret_cast<uintptr_t>(target));
|
|
// Explicit abort to let compilers know this code-path does not return
|
|
abort();
|
|
}
|
|
#endif
|
|
return *target;
|
|
}
|
|
|
|
template <typename T>
|
|
static inline const T& RefAt(const void* x, size_t offset) {
|
|
const T* target =
|
|
reinterpret_cast<const T*>(static_cast<const char*>(x) + offset);
|
|
#if !defined(NDEBUG) && !(defined(_MSC_VER) && defined(_M_IX86))
|
|
// Check the alignment in debug mode, except in 32-bit msvc because it does
|
|
// not respect the alignment as expressed by `alignof(T)`
|
|
if (PROTOBUF_PREDICT_FALSE(
|
|
reinterpret_cast<uintptr_t>(target) % alignof(T) != 0)) {
|
|
AlignFail(std::integral_constant<size_t, alignof(T)>(),
|
|
reinterpret_cast<uintptr_t>(target));
|
|
// Explicit abort to let compilers know this code-path does not return
|
|
abort();
|
|
}
|
|
#endif
|
|
return *target;
|
|
}
|
|
|
|
static const TcParseTableBase* GetTableFromAux(
|
|
uint16_t type_card, TcParseTableBase::FieldAux aux);
|
|
static MessageLite* NewMessage(const TcParseTableBase* table, Arena* arena);
|
|
static MessageLite* AddMessage(const TcParseTableBase* table,
|
|
RepeatedPtrFieldBase& field);
|
|
|
|
template <typename T, bool is_split>
|
|
static inline T& MaybeCreateRepeatedRefAt(void* x, size_t offset,
|
|
MessageLite* msg) {
|
|
if (!is_split) return RefAt<T>(x, offset);
|
|
void*& ptr = RefAt<void*>(x, offset);
|
|
if (ptr == DefaultRawPtr()) {
|
|
ptr = Arena::Create<T>(msg->GetArena());
|
|
}
|
|
return *static_cast<T*>(ptr);
|
|
}
|
|
|
|
template <typename T, bool is_split>
|
|
static inline RepeatedField<T>& MaybeCreateRepeatedFieldRefAt(
|
|
void* x, size_t offset, MessageLite* msg) {
|
|
return MaybeCreateRepeatedRefAt<RepeatedField<T>, is_split>(x, offset, msg);
|
|
}
|
|
|
|
template <typename T, bool is_split>
|
|
static inline RepeatedPtrField<T>& MaybeCreateRepeatedPtrFieldRefAt(
|
|
void* x, size_t offset, MessageLite* msg) {
|
|
return MaybeCreateRepeatedRefAt<RepeatedPtrField<T>, is_split>(x, offset,
|
|
msg);
|
|
}
|
|
|
|
template <typename T>
|
|
static inline T ReadAt(const void* x, size_t offset) {
|
|
T out;
|
|
memcpy(&out, static_cast<const char*>(x) + offset, sizeof(T));
|
|
return out;
|
|
}
|
|
|
|
// Mini parsing:
|
|
//
|
|
// This function parses a field from incoming data based on metadata stored in
|
|
// the message definition. If the field is not defined in the message, it is
|
|
// stored in either the ExtensionSet (if applicable) or the UnknownFieldSet.
|
|
//
|
|
// NOTE: Currently, this function only calls the table-level fallback
|
|
// function, so it should only be called as the fallback from fast table
|
|
// parsing.
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MiniParse(
|
|
PROTOBUF_TC_PARAM_NO_DATA_DECL);
|
|
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEndG1(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastEndG2(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// For `map` mini parsing generate a type card for the key/value.
|
|
template <typename MapField>
|
|
static constexpr MapAuxInfo GetMapAuxInfo(bool fail_on_utf8_failure,
|
|
bool log_debug_utf8_failure,
|
|
bool validated_enum_value,
|
|
int key_type, int value_type) {
|
|
using MapType = typename MapField::MapType;
|
|
using Node = typename MapType::Node;
|
|
static_assert(alignof(Node) == alignof(NodeBase), "");
|
|
// Verify the assumption made in MpMap, guaranteed by Map<>.
|
|
assert(PROTOBUF_FIELD_OFFSET(Node, kv.first) == sizeof(NodeBase));
|
|
return {
|
|
MakeMapTypeCard(static_cast<WireFormatLite::FieldType>(key_type)),
|
|
MakeMapTypeCard(static_cast<WireFormatLite::FieldType>(value_type)),
|
|
true,
|
|
!std::is_base_of<MapFieldBaseForParse, MapField>::value,
|
|
fail_on_utf8_failure,
|
|
log_debug_utf8_failure,
|
|
validated_enum_value,
|
|
Node::size_info(),
|
|
};
|
|
}
|
|
|
|
private:
|
|
// Optimized small tag varint parser for int32/int64
|
|
template <typename FieldType>
|
|
PROTOBUF_CC static const char* FastVarintS1(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
friend class GeneratedTcTableLiteTest;
|
|
static void* MaybeGetSplitBase(MessageLite* msg, bool is_split,
|
|
const TcParseTableBase* table);
|
|
|
|
// Test only access to verify that the right function is being called via
|
|
// MiniParse.
|
|
struct TestMiniParseResult {
|
|
TailCallParseFunc called_func;
|
|
uint32_t tag;
|
|
const TcParseTableBase::FieldEntry* found_entry;
|
|
const char* ptr;
|
|
};
|
|
PROTOBUF_NOINLINE
|
|
static TestMiniParseResult TestMiniParse(PROTOBUF_TC_PARAM_DECL);
|
|
template <bool export_called_function>
|
|
PROTOBUF_CC static const char* MiniParse(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
template <typename TagType, bool group_coding, bool aux_is_table>
|
|
PROTOBUF_CC static inline const char* SingularParseMessageAuxImpl(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType, bool group_coding, bool aux_is_table>
|
|
PROTOBUF_CC static inline const char* RepeatedParseMessageAuxImpl(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType>
|
|
PROTOBUF_CC static inline const char* LazyMessage(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
template <typename TagType>
|
|
PROTOBUF_CC static const char* FastEndGroupImpl(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits(
|
|
MessageLite* msg, uint64_t hasbits, const TcParseTableBase* table) {
|
|
const uint32_t has_bits_offset = table->has_bits_offset;
|
|
if (has_bits_offset) {
|
|
// Only the first 32 has-bits are updated. Nothing above those is stored,
|
|
// but e.g. messages without has-bits update the upper bits.
|
|
RefAt<uint32_t>(msg, has_bits_offset) |= static_cast<uint32_t>(hasbits);
|
|
}
|
|
}
|
|
|
|
PROTOBUF_CC static const char* TagDispatch(PROTOBUF_TC_PARAM_NO_DATA_DECL);
|
|
PROTOBUF_CC static const char* ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_DECL);
|
|
PROTOBUF_CC static const char* ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_DECL);
|
|
PROTOBUF_NOINLINE
|
|
PROTOBUF_CC static const char* Error(PROTOBUF_TC_PARAM_NO_DATA_DECL);
|
|
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* FastUnknownEnumFallback(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE
|
|
PROTOBUF_CC static const char* MpUnknownEnumFallback(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
class ScopedArenaSwap;
|
|
|
|
struct UnknownFieldOps {
|
|
void (*write_varint)(MessageLite* msg, int number, int value);
|
|
void (*write_length_delimited)(MessageLite* msg, int number,
|
|
absl::string_view value);
|
|
};
|
|
|
|
static const UnknownFieldOps& GetUnknownFieldOps(
|
|
const TcParseTableBase* table);
|
|
|
|
template <typename UnknownFieldsT>
|
|
static void WriteVarintToUnknown(MessageLite* msg, int number, int value) {
|
|
internal::WriteVarint(
|
|
number, value,
|
|
msg->_internal_metadata_.mutable_unknown_fields<UnknownFieldsT>());
|
|
}
|
|
template <typename UnknownFieldsT>
|
|
static void WriteLengthDelimitedToUnknown(MessageLite* msg, int number,
|
|
absl::string_view value) {
|
|
internal::WriteLengthDelimited(
|
|
number, value,
|
|
msg->_internal_metadata_.mutable_unknown_fields<UnknownFieldsT>());
|
|
}
|
|
|
|
template <class MessageBaseT, class UnknownFieldsT>
|
|
PROTOBUF_CC static const char* GenericFallbackImpl(PROTOBUF_TC_PARAM_DECL) {
|
|
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) {
|
|
// This is the ABI used by GetUnknownFieldOps(). Return the vtable.
|
|
static constexpr UnknownFieldOps kOps = {
|
|
WriteVarintToUnknown<UnknownFieldsT>,
|
|
WriteLengthDelimitedToUnknown<UnknownFieldsT>};
|
|
return reinterpret_cast<const char*>(&kOps);
|
|
}
|
|
|
|
SyncHasbits(msg, hasbits, table);
|
|
uint32_t tag = data.tag();
|
|
if ((tag & 7) == WireFormatLite::WIRETYPE_END_GROUP || tag == 0) {
|
|
ctx->SetLastTag(tag);
|
|
return ptr;
|
|
}
|
|
|
|
if (table->extension_offset != 0) {
|
|
// We don't need to check the extension ranges. If it is not an extension
|
|
// it will be handled just like if it was an unknown extension: sent to
|
|
// the unknown field set.
|
|
return RefAt<ExtensionSet>(msg, table->extension_offset)
|
|
.ParseField(
|
|
tag, ptr,
|
|
static_cast<const MessageBaseT*>(table->default_instance()),
|
|
&msg->_internal_metadata_, ctx);
|
|
} else {
|
|
// Otherwise, we directly put it on the unknown field set.
|
|
return UnknownFieldParse(
|
|
tag,
|
|
msg->_internal_metadata_.mutable_unknown_fields<UnknownFieldsT>(),
|
|
ptr, ctx);
|
|
}
|
|
}
|
|
|
|
template <class MessageBaseT>
|
|
PROTOBUF_CC static const char* MessageSetWireFormatParseLoopImpl(
|
|
PROTOBUF_TC_PARAM_NO_DATA_DECL) {
|
|
return RefAt<ExtensionSet>(msg, table->extension_offset)
|
|
.ParseMessageSet(
|
|
ptr, static_cast<const MessageBaseT*>(table->default_instance()),
|
|
&msg->_internal_metadata_, ctx);
|
|
}
|
|
|
|
// Note: `inline` is needed on template function declarations below to avoid
|
|
// -Wattributes diagnostic in GCC.
|
|
|
|
// Implementations for fast fixed field parsing functions:
|
|
template <typename LayoutType, typename TagType>
|
|
PROTOBUF_CC static inline const char* SingularFixed(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename LayoutType, typename TagType>
|
|
PROTOBUF_CC static inline const char* RepeatedFixed(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename LayoutType, typename TagType>
|
|
PROTOBUF_CC static inline const char* PackedFixed(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Implementations for fast varint field parsing functions:
|
|
template <typename FieldType, typename TagType, bool zigzag = false>
|
|
PROTOBUF_CC static inline const char* SingularVarint(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename FieldType, typename TagType, bool zigzag = false>
|
|
PROTOBUF_CC static inline const char* RepeatedVarint(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename FieldType, typename TagType, bool zigzag = false>
|
|
PROTOBUF_CC static inline const char* PackedVarint(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Helper for ints > 127:
|
|
template <typename FieldType, typename TagType, bool zigzag = false>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* SingularVarBigint(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Implementations for fast enum field parsing functions:
|
|
template <typename TagType, uint16_t xform_val>
|
|
PROTOBUF_CC static inline const char* SingularEnum(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType, uint8_t min>
|
|
PROTOBUF_CC static inline const char* SingularEnumSmallRange(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType, uint16_t xform_val>
|
|
PROTOBUF_CC static inline const char* RepeatedEnum(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType, uint16_t xform_val>
|
|
PROTOBUF_CC static inline const char* PackedEnum(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType, uint8_t min>
|
|
PROTOBUF_CC static inline const char* RepeatedEnumSmallRange(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType, uint8_t min>
|
|
PROTOBUF_CC static inline const char* PackedEnumSmallRange(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
|
|
// Implementations for fast string field parsing functions:
|
|
enum Utf8Type { kNoUtf8 = 0, kUtf8 = 1, kUtf8ValidateOnly = 2 };
|
|
template <typename TagType, typename FieldType, Utf8Type utf8>
|
|
PROTOBUF_CC static inline const char* SingularString(PROTOBUF_TC_PARAM_DECL);
|
|
template <typename TagType, typename FieldType, Utf8Type utf8>
|
|
PROTOBUF_CC static inline const char* RepeatedString(PROTOBUF_TC_PARAM_DECL);
|
|
|
|
static inline const char* ParseRepeatedStringOnce(
|
|
const char* ptr, SerialArena* serial_arena, ParseContext* ctx,
|
|
RepeatedPtrField<std::string>& field);
|
|
|
|
PROTOBUF_NOINLINE
|
|
static void AddUnknownEnum(MessageLite* msg, const TcParseTableBase* table,
|
|
uint32_t tag, int32_t enum_value);
|
|
|
|
static void WriteMapEntryAsUnknown(MessageLite* msg,
|
|
const TcParseTableBase* table,
|
|
uint32_t tag, NodeBase* node,
|
|
MapAuxInfo map_info);
|
|
|
|
static void InitializeMapNodeEntry(void* obj, MapTypeCard type_card,
|
|
UntypedMapBase& map,
|
|
const TcParseTableBase::FieldAux* aux,
|
|
bool is_key);
|
|
PROTOBUF_NOINLINE
|
|
static void DestroyMapNode(NodeBase* node, MapAuxInfo map_info,
|
|
UntypedMapBase& map);
|
|
static const char* ParseOneMapEntry(NodeBase* node, const char* ptr,
|
|
ParseContext* ctx,
|
|
const TcParseTableBase::FieldAux* aux,
|
|
const TcParseTableBase* table,
|
|
const TcParseTableBase::FieldEntry& entry,
|
|
Arena* arena);
|
|
|
|
// Mini field lookup:
|
|
static const TcParseTableBase::FieldEntry* FindFieldEntry(
|
|
const TcParseTableBase* table, uint32_t field_num);
|
|
static absl::string_view MessageName(const TcParseTableBase* table);
|
|
static absl::string_view FieldName(const TcParseTableBase* table,
|
|
const TcParseTableBase::FieldEntry*);
|
|
static bool ChangeOneof(const TcParseTableBase* table,
|
|
const TcParseTableBase::FieldEntry& entry,
|
|
uint32_t field_num, ParseContext* ctx,
|
|
MessageLite* msg);
|
|
|
|
// UTF-8 validation:
|
|
static void ReportFastUtf8Error(uint32_t decoded_tag,
|
|
const TcParseTableBase* table);
|
|
static bool MpVerifyUtf8(absl::string_view wire_bytes,
|
|
const TcParseTableBase* table,
|
|
const TcParseTableBase::FieldEntry& entry,
|
|
uint16_t xform_val);
|
|
static bool MpVerifyUtf8(const absl::Cord& wire_bytes,
|
|
const TcParseTableBase* table,
|
|
const TcParseTableBase::FieldEntry& entry,
|
|
uint16_t xform_val);
|
|
|
|
// For FindFieldEntry tests:
|
|
friend class FindFieldEntryTest;
|
|
friend struct ParseFunctionGeneratorTestPeer;
|
|
friend struct FuzzPeer;
|
|
static constexpr const uint32_t kMtSmallScanSize = 4;
|
|
|
|
// Mini parsing:
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpVarint(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpRepeatedVarint(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split, typename FieldType, uint16_t xform_val>
|
|
PROTOBUF_CC static const char* MpRepeatedVarintT(PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpPackedVarint(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split, typename FieldType, uint16_t xform_val>
|
|
PROTOBUF_CC static const char* MpPackedVarintT(PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpFixed(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpRepeatedFixed(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpPackedFixed(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpString(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpRepeatedString(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpMessage(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split, bool is_group>
|
|
PROTOBUF_CC static const char* MpRepeatedMessageOrGroup(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_CC static const char* MpLazyMessage(PROTOBUF_TC_PARAM_DECL);
|
|
PROTOBUF_NOINLINE
|
|
PROTOBUF_CC static const char* MpFallback(PROTOBUF_TC_PARAM_DECL);
|
|
template <bool is_split>
|
|
PROTOBUF_NOINLINE PROTOBUF_CC static const char* MpMap(
|
|
PROTOBUF_TC_PARAM_DECL);
|
|
};
|
|
|
|
// Dispatch to the designated parse function
|
|
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::TagDispatch(
|
|
PROTOBUF_TC_PARAM_NO_DATA_DECL) {
|
|
const auto coded_tag = UnalignedLoad<uint16_t>(ptr);
|
|
const size_t idx = coded_tag & table->fast_idx_mask;
|
|
PROTOBUF_ASSUME((idx & 7) == 0);
|
|
auto* fast_entry = table->fast_entry(idx >> 3);
|
|
TcFieldData data = fast_entry->bits;
|
|
data.data ^= coded_tag;
|
|
PROTOBUF_MUSTTAIL return fast_entry->target()(PROTOBUF_TC_PARAM_PASS);
|
|
}
|
|
|
|
// We can only safely call from field to next field if the call is optimized
|
|
// to a proper tail call. Otherwise we blow through stack. Clang and gcc
|
|
// reliably do this optimization in opt mode, but do not perform this in debug
|
|
// mode. Luckily the structure of the algorithm is such that it's always
|
|
// possible to just return and use the enclosing parse loop as a trampoline.
|
|
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToTagDispatch(
|
|
PROTOBUF_TC_PARAM_NO_DATA_DECL) {
|
|
constexpr bool always_return = !PROTOBUF_TAILCALL;
|
|
if (always_return || !ctx->DataAvailable(ptr)) {
|
|
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
|
|
}
|
|
PROTOBUF_MUSTTAIL return TagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS);
|
|
}
|
|
|
|
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ToParseLoop(
|
|
PROTOBUF_TC_PARAM_NO_DATA_DECL) {
|
|
(void)ctx;
|
|
SyncHasbits(msg, hasbits, table);
|
|
return ptr;
|
|
}
|
|
|
|
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::ParseLoop(
|
|
MessageLite* msg, const char* ptr, ParseContext* ctx,
|
|
const TcParseTableBase* table) {
|
|
// Note: TagDispatch uses a dispatch table at "&table->fast_entries".
|
|
// For fast dispatch, we'd like to have a pointer to that, but if we use
|
|
// that expression, there's no easy way to get back to "table", which we also
|
|
// need during dispatch. It turns out that "table + 1" points exactly to
|
|
// fast_entries, so we just increment table by 1 here, to get the register
|
|
// holding the value we want.
|
|
table += 1;
|
|
while (!ctx->Done(&ptr)) {
|
|
#if defined(__GNUC__)
|
|
// Note: this asm prevents the compiler (clang, specifically) from
|
|
// believing (thanks to CSE) that it needs to dedicate a register both
|
|
// to "table" and "&table->fast_entries".
|
|
// TODO: remove this asm
|
|
asm("" : "+r"(table));
|
|
#endif
|
|
ptr = TagDispatch(msg, ptr, ctx, TcFieldData::DefaultInit(), table - 1, 0);
|
|
if (ptr == nullptr) break;
|
|
if (ctx->LastTag() != 1) break; // Ended on terminating tag
|
|
}
|
|
table -= 1;
|
|
if (ABSL_PREDICT_FALSE(table->has_post_loop_handler)) {
|
|
return table->post_loop_handler(msg, ptr, ctx);
|
|
}
|
|
return ptr;
|
|
}
|
|
|
|
// Prints the type card as or of labels, using known higher level labels.
|
|
// Used for code generation, but also useful for debugging.
|
|
PROTOBUF_EXPORT std::string TypeCardToString(uint16_t type_card);
|
|
|
|
} // namespace internal
|
|
} // namespace protobuf
|
|
} // namespace google
|
|
|
|
#include "google/protobuf/port_undef.inc"
|
|
|
|
#endif // GOOGLE_PROTOBUF_GENERATED_MESSAGE_TCTABLE_IMPL_H__
|