Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ARROW-10479: [C++] Initial support of low bitness Decimals (16, 32, 64) #8578

Closed
wants to merge 8 commits into from
Closed

ARROW-10479: [C++] Initial support of low bitness Decimals (16, 32, 64) #8578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1fa2d20
ARROW-10479: Get rid of code duplication at decimal type builders
dchigarev Oct 9, 2020
b6f3cec
Draft implementation of low bitness decimals for C++
dchigarev Nov 10, 2020
4d15198
Added decimal convertions from/to python
dchigarev Dec 28, 2020
a108f38
Attempt to fix linking error on Windows
dchigarev Jan 14, 2021
0c82404
Edited R datatypes enum
dchigarev Jan 15, 2021
91ed793
Tests fix
dchigarev Jan 15, 2021
ba94f9c
Lint fixes
dchigarev Jan 15, 2021
b22cdfd
Tests fixed
dchigarev Jan 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions cpp/src/arrow/array/array_base.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,18 @@ struct ScalarFromArraySlotImpl {
return Finish(a.Value(index_));
}

Status Visit(const Decimal128Array& a) {
return Finish(Decimal128(a.GetValue(index_)));
#define DECL_DECIMAL_VISIT(width) \
Status Visit(const Decimal##width##Array& a) { \
return Finish(Decimal##width(a.GetValue(index_))); \
}

Status Visit(const Decimal256Array& a) {
return Finish(Decimal256(a.GetValue(index_)));
}
DECL_DECIMAL_VISIT(16)
DECL_DECIMAL_VISIT(32)
DECL_DECIMAL_VISIT(64)
DECL_DECIMAL_VISIT(128)
DECL_DECIMAL_VISIT(256)

#undef DECL_DECIMAL_VISIT

template <typename T>
Status Visit(const BaseBinaryArray<T>& a) {
Expand Down
33 changes: 12 additions & 21 deletions cpp/src/arrow/array/array_decimal.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,32 +32,23 @@ namespace arrow {

using internal::checked_cast;

// ----------------------------------------------------------------------
// Decimal128

Decimal128Array::Decimal128Array(const std::shared_ptr<ArrayData>& data)
template <uint32_t width>
BaseDecimalArray<width>::BaseDecimalArray(const std::shared_ptr<ArrayData>& data)
: FixedSizeBinaryArray(data) {
ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL128);
ARROW_CHECK_EQ(data->type->id(), DecimalTypeTraits<width>::Id);
}

std::string Decimal128Array::FormatValue(int64_t i) const {
const auto& type_ = checked_cast<const Decimal128Type&>(*type());
const Decimal128 value(GetValue(i));
template <uint32_t width>
std::string BaseDecimalArray<width>::FormatValue(int64_t i) const {
const auto& type_ = checked_cast<const TypeClass&>(*type());
const ValueType value(GetValue(i));
return value.ToString(type_.scale());
}

// ----------------------------------------------------------------------
// Decimal256

Decimal256Array::Decimal256Array(const std::shared_ptr<ArrayData>& data)
: FixedSizeBinaryArray(data) {
ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL256);
}

std::string Decimal256Array::FormatValue(int64_t i) const {
const auto& type_ = checked_cast<const Decimal256Type&>(*type());
const Decimal256 value(GetValue(i));
return value.ToString(type_.scale());
}
template class ARROW_EXPORT BaseDecimalArray<16>;
template class ARROW_EXPORT BaseDecimalArray<32>;
template class ARROW_EXPORT BaseDecimalArray<64>;
template class ARROW_EXPORT BaseDecimalArray<128>;
template class ARROW_EXPORT BaseDecimalArray<256>;

} // namespace arrow
34 changes: 9 additions & 25 deletions cpp/src/arrow/array/array_decimal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,43 +24,27 @@
#include "arrow/array/array_binary.h"
#include "arrow/array/data.h"
#include "arrow/type.h"
#include "arrow/util/decimal_type_traits.h"
#include "arrow/util/visibility.h"

namespace arrow {

// ----------------------------------------------------------------------
// Decimal128Array

/// Concrete Array class for 128-bit decimal data
class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray {
/// Template Array class for decimal data
template <uint32_t width>
class ARROW_EXPORT BaseDecimalArray : public FixedSizeBinaryArray {
public:
using TypeClass = Decimal128Type;
using TypeClass = typename DecimalTypeTraits<width>::TypeClass;
using ValueType = typename DecimalTypeTraits<width>::ValueType;

using FixedSizeBinaryArray::FixedSizeBinaryArray;

/// \brief Construct Decimal128Array from ArrayData instance
explicit Decimal128Array(const std::shared_ptr<ArrayData>& data);
/// \brief Construct DecimalArray from ArrayData instance
explicit BaseDecimalArray(const std::shared_ptr<ArrayData>& data);

std::string FormatValue(int64_t i) const;
};

// Backward compatibility
using DecimalArray = Decimal128Array;

// ----------------------------------------------------------------------
// Decimal256Array

/// Concrete Array class for 256-bit decimal data
class ARROW_EXPORT Decimal256Array : public FixedSizeBinaryArray {
public:
using TypeClass = Decimal256Type;

using FixedSizeBinaryArray::FixedSizeBinaryArray;

/// \brief Construct Decimal256Array from ArrayData instance
explicit Decimal256Array(const std::shared_ptr<ArrayData>& data);

std::string FormatValue(int64_t i) const;
};
using DecimalArray = BaseDecimalArray<128>;

} // namespace arrow
12 changes: 12 additions & 0 deletions cpp/src/arrow/array/array_dict_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -859,6 +859,18 @@ void TestDecimalDictionaryBuilderBasic(std::shared_ptr<DataType> decimal_type) {
ASSERT_TRUE(expected.Equals(result));
}

TEST(TestDecimal16DictionaryBuilder, Basic) {
TestDecimalDictionaryBuilderBasic<Decimal16>(arrow::decimal16(2, 0));
}

TEST(TestDecimal32DictionaryBuilder, Basic) {
TestDecimalDictionaryBuilderBasic<Decimal32>(arrow::decimal32(2, 0));
}

TEST(TestDecimal64DictionaryBuilder, Basic) {
TestDecimalDictionaryBuilderBasic<Decimal64>(arrow::decimal64(2, 0));
}

TEST(TestDecimal128DictionaryBuilder, Basic) {
TestDecimalDictionaryBuilderBasic<Decimal128>(arrow::decimal128(2, 0));
}
Expand Down
122 changes: 78 additions & 44 deletions cpp/src/arrow/array/array_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2407,6 +2407,34 @@ class DecimalTest : public ::testing::TestWithParam<int> {
}
}

void InitNoNullsTest(int32_t precision) {
std::vector<DecimalValue> draw = {DecimalValue(1), DecimalValue(-2),
DecimalValue(2389), DecimalValue(4),
DecimalValue(-12348)};
std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
this->TestCreate(precision, draw, valid_bytes, 0);
this->TestCreate(precision, draw, valid_bytes, 2);
}

void InitWithNullsTest(int32_t precision, std::string big_value,
std::string big_negate_value) {
std::vector<DecimalValue> draw = {DecimalValue(1), DecimalValue(2), DecimalValue(-1),
DecimalValue(4), DecimalValue(-1), DecimalValue(1),
DecimalValue(2)};
DecimalValue big;
ASSERT_OK_AND_ASSIGN(big, DecimalValue::FromString(big_value));
draw.push_back(big);

DecimalValue big_negative;
ASSERT_OK_AND_ASSIGN(big_negative, DecimalValue::FromString(big_negate_value));
draw.push_back(big_negative);

std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
true, true, true, true};
this->TestCreate(precision, draw, valid_bytes, 0);
this->TestCreate(precision, draw, valid_bytes, 2);
}

template <size_t BYTE_WIDTH = 16>
void TestCreate(int32_t precision, const DecimalVector& draw,
const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
Expand Down Expand Up @@ -2451,34 +2479,58 @@ class DecimalTest : public ::testing::TestWithParam<int> {
}
};

using Decimal16Test = DecimalTest<Decimal16Type>;

TEST_P(Decimal16Test, NoNulls) {
int32_t precision = GetParam();
this->InitNoNullsTest(precision);
}

TEST_P(Decimal16Test, WithNulls) {
int32_t precision = GetParam();
this->InitWithNullsTest(precision, "163.84", "-163.84");
}

INSTANTIATE_TEST_SUITE_P(Decimal16Test, Decimal16Test, ::testing::Range(1, 5));

using Decimal32Test = DecimalTest<Decimal32Type>;

TEST_P(Decimal32Test, NoNulls) {
int32_t precision = GetParam();
this->InitNoNullsTest(precision);
}

TEST_P(Decimal32Test, WithNulls) {
int32_t precision = GetParam();
this->InitWithNullsTest(precision, "107374.1824", "-107374.1824");
}

INSTANTIATE_TEST_SUITE_P(Decimal32Test, Decimal32Test, ::testing::Range(1, 10));

using Decimal64Test = DecimalTest<Decimal64Type>;

TEST_P(Decimal64Test, NoNulls) {
int32_t precision = GetParam();
this->InitNoNullsTest(precision);
}

TEST_P(Decimal64Test, WithNulls) {
int32_t precision = GetParam();
this->InitWithNullsTest(precision, "46116860184.27387904", "-46116860184.27387904");
}

INSTANTIATE_TEST_SUITE_P(Decimal64Test, Decimal64Test, ::testing::Range(1, 19));

using Decimal128Test = DecimalTest<Decimal128Type>;

TEST_P(Decimal128Test, NoNulls) {
int32_t precision = GetParam();
std::vector<Decimal128> draw = {Decimal128(1), Decimal128(-2), Decimal128(2389),
Decimal128(4), Decimal128(-12348)};
std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
this->TestCreate(precision, draw, valid_bytes, 0);
this->TestCreate(precision, draw, valid_bytes, 2);
this->InitNoNullsTest(precision);
}

TEST_P(Decimal128Test, WithNulls) {
int32_t precision = GetParam();
std::vector<Decimal128> draw = {Decimal128(1), Decimal128(2), Decimal128(-1),
Decimal128(4), Decimal128(-1), Decimal128(1),
Decimal128(2)};
Decimal128 big;
ASSERT_OK_AND_ASSIGN(big, Decimal128::FromString("230342903942.234234"));
draw.push_back(big);

Decimal128 big_negative;
ASSERT_OK_AND_ASSIGN(big_negative, Decimal128::FromString("-23049302932.235234"));
draw.push_back(big_negative);

std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
true, true, true, true};
this->TestCreate(precision, draw, valid_bytes, 0);
this->TestCreate(precision, draw, valid_bytes, 2);
this->InitWithNullsTest(precision, "23049302932.235234", "-23049302932.235234");
}

INSTANTIATE_TEST_SUITE_P(Decimal128Test, Decimal128Test, ::testing::Range(1, 38));
Expand All @@ -2487,34 +2539,16 @@ using Decimal256Test = DecimalTest<Decimal256Type>;

TEST_P(Decimal256Test, NoNulls) {
int32_t precision = GetParam();
std::vector<Decimal256> draw = {Decimal256(1), Decimal256(-2), Decimal256(2389),
Decimal256(4), Decimal256(-12348)};
std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
this->TestCreate(precision, draw, valid_bytes, 0);
this->TestCreate(precision, draw, valid_bytes, 2);
this->InitNoNullsTest(precision);
}

TEST_P(Decimal256Test, WithNulls) {
int32_t precision = GetParam();
std::vector<Decimal256> draw = {Decimal256(1), Decimal256(2), Decimal256(-1),
Decimal256(4), Decimal256(-1), Decimal256(1),
Decimal256(2)};
Decimal256 big; // (pow(2, 255) - 1) / pow(10, 38)
ASSERT_OK_AND_ASSIGN(big,
Decimal256::FromString("578960446186580977117854925043439539266."
"34992332820282019728792003956564819967"));
draw.push_back(big);

Decimal256 big_negative; // -pow(2, 255) / pow(10, 38)
ASSERT_OK_AND_ASSIGN(big_negative,
Decimal256::FromString("-578960446186580977117854925043439539266."
"34992332820282019728792003956564819968"));
draw.push_back(big_negative);

std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
true, true, true, true};
this->TestCreate(precision, draw, valid_bytes, 0);
this->TestCreate(precision, draw, valid_bytes, 2);
this->InitWithNullsTest(precision,
"578960446186580977117854925043439539266."
"34992332820282019728792003956564819967",
"-578960446186580977117854925043439539266."
"34992332820282019728792003956564819968");
}

INSTANTIATE_TEST_SUITE_P(Decimal256Test, Decimal256Test,
Expand Down
62 changes: 19 additions & 43 deletions cpp/src/arrow/array/builder_decimal.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,30 +33,35 @@ class Buffer;
class MemoryPool;

// ----------------------------------------------------------------------
// Decimal128Builder
// BaseDecimalBuilder

Decimal128Builder::Decimal128Builder(const std::shared_ptr<DataType>& type,
MemoryPool* pool)
template <uint32_t width>
BaseDecimalBuilder<width>::BaseDecimalBuilder(const std::shared_ptr<DataType>& type,
MemoryPool* pool)
: FixedSizeBinaryBuilder(type, pool),
decimal_type_(internal::checked_pointer_cast<Decimal128Type>(type)) {}
decimal_type_(internal::checked_pointer_cast<TypeClass>(type)) {}

Status Decimal128Builder::Append(Decimal128 value) {
template <uint32_t width>
Status BaseDecimalBuilder<width>::Append(ValueType value) {
RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
UnsafeAppend(value);
return Status::OK();
}

void Decimal128Builder::UnsafeAppend(Decimal128 value) {
template <uint32_t width>
void BaseDecimalBuilder<width>::UnsafeAppend(ValueType value) {
value.ToBytes(GetMutableValue(length()));
byte_builder_.UnsafeAdvance(16);
byte_builder_.UnsafeAdvance((width >> 3));
UnsafeAppendToBitmap(true);
}

void Decimal128Builder::UnsafeAppend(util::string_view value) {
template <uint32_t width>
void BaseDecimalBuilder<width>::UnsafeAppend(util::string_view value) {
FixedSizeBinaryBuilder::UnsafeAppend(value);
}

Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
template <uint32_t width>
Status BaseDecimalBuilder<width>::FinishInternal(std::shared_ptr<ArrayData>* out) {
std::shared_ptr<Buffer> data;
RETURN_NOT_OK(byte_builder_.Finish(&data));
std::shared_ptr<Buffer> null_bitmap;
Expand All @@ -67,39 +72,10 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
return Status::OK();
}

// ----------------------------------------------------------------------
// Decimal256Builder

Decimal256Builder::Decimal256Builder(const std::shared_ptr<DataType>& type,
MemoryPool* pool)
: FixedSizeBinaryBuilder(type, pool),
decimal_type_(internal::checked_pointer_cast<Decimal256Type>(type)) {}

Status Decimal256Builder::Append(const Decimal256& value) {
RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
UnsafeAppend(value);
return Status::OK();
}

void Decimal256Builder::UnsafeAppend(const Decimal256& value) {
value.ToBytes(GetMutableValue(length()));
byte_builder_.UnsafeAdvance(32);
UnsafeAppendToBitmap(true);
}

void Decimal256Builder::UnsafeAppend(util::string_view value) {
FixedSizeBinaryBuilder::UnsafeAppend(value);
}

Status Decimal256Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
std::shared_ptr<Buffer> data;
RETURN_NOT_OK(byte_builder_.Finish(&data));
std::shared_ptr<Buffer> null_bitmap;
RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap));

*out = ArrayData::Make(type(), length_, {null_bitmap, data}, null_count_);
capacity_ = length_ = null_count_ = 0;
return Status::OK();
}
template class ARROW_EXPORT BaseDecimalBuilder<16>;
template class ARROW_EXPORT BaseDecimalBuilder<32>;
template class ARROW_EXPORT BaseDecimalBuilder<64>;
template class ARROW_EXPORT BaseDecimalBuilder<128>;
template class ARROW_EXPORT BaseDecimalBuilder<256>;

} // namespace arrow
Loading