diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e847651369..9cf2b3bd3af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,10 @@ * None. ### Breaking changes -* None. +* Support for upgrading from Realm files produced by RealmCore v5.23.9 or earlier is no longer supported. ### Compatibility -* Fileformat: Generates files with format v23. Reads and automatically upgrade from fileformat v5. +* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. ----------- diff --git a/doc/CHANGELOG_template.md b/doc/CHANGELOG_template.md index 509b81e4091..fc302b0723b 100644 --- a/doc/CHANGELOG_template.md +++ b/doc/CHANGELOG_template.md @@ -12,7 +12,7 @@ * None. ### Compatibility -* Fileformat: Generates files with format v23. Reads and automatically upgrade from fileformat v5. +* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. ----------- diff --git a/src/realm/backup_restore.cpp b/src/realm/backup_restore.cpp index a24d8a98b2d..c395246c5e5 100644 --- a/src/realm/backup_restore.cpp +++ b/src/realm/backup_restore.cpp @@ -34,14 +34,13 @@ using VersionList = BackupHandler::VersionList; using VersionTimeList = BackupHandler::VersionTimeList; // Note: accepted versions should have new versions added at front -const VersionList BackupHandler::accepted_versions_ = {23, 22, 21, 20, 11, 10, 9, 8, 7, 6, 5, 0}; +const VersionList BackupHandler::accepted_versions_ = {24, 23, 22, 21, 20, 11, 10}; // the pair is // we keep backup files in 3 months. static constexpr int three_months = 3 * 31 * 24 * 60 * 60; -const VersionTimeList BackupHandler::delete_versions_{ - {22, three_months}, {21, three_months}, {20, three_months}, {11, three_months}, {10, three_months}, - {9, three_months}, {8, three_months}, {7, three_months}, {6, three_months}, {5, three_months}}; +const VersionTimeList BackupHandler::delete_versions_{{23, three_months}, {22, three_months}, {21, three_months}, + {20, three_months}, {11, three_months}, {10, three_months}}; // helper functions diff --git a/src/realm/group.cpp b/src/realm/group.cpp index 6f8636ff9a2..339dd6ae486 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -401,6 +401,7 @@ int Group::read_only_version_check(SlabAlloc& alloc, ref_type top_ref, const std case 0: file_format_ok = (top_ref == 0); break; + case 23: case g_current_file_format_version: file_format_ok = true; break; diff --git a/src/realm/group.hpp b/src/realm/group.hpp index 43ac34f9d4c..db1ed3533d7 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -788,6 +788,8 @@ class Group : public ArrayParent { /// /// 23 Layout of Set and Dictionary changed. /// + /// 24 Variable sized arrays for Decimal128. + /// /// IMPORTANT: When introducing a new file format version, be sure to review /// the file validity checks in Group::open() and DB::do_open, the file /// format selection logic in @@ -795,7 +797,7 @@ class Group : public ArrayParent { /// upgrade logic in Group::upgrade_file_format(), AND the lists of accepted /// file formats and the version deletion list residing in "backup_restore.cpp" - static constexpr int g_current_file_format_version = 23; + static constexpr int g_current_file_format_version = 24; int get_file_format_version() const noexcept; void set_file_format_version(int) noexcept; diff --git a/src/realm/table.cpp b/src/realm/table.cpp index 75f6134372d..6eec41b7843 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -1320,689 +1320,8 @@ bool Table::has_search_index(ColKey col_key) const noexcept return m_index_accessors[col_key.get_index().val] != nullptr; } -void Table::migrate_column_info() -{ - bool changes = false; - TableKey tk = (get_name() == "pk") ? TableKey(0) : get_key(); - changes |= m_spec.convert_column_attributes(); - changes |= m_spec.convert_column_keys(tk); - - if (changes) { - build_column_mapping(); - } -} - -bool Table::verify_column_keys() -{ - size_t nb_public_columns = m_spec.get_public_column_count(); - size_t nb_columns = m_spec.get_column_count(); - bool modified = false; - - auto check = [&]() { - for (size_t spec_ndx = nb_public_columns; spec_ndx < nb_columns; spec_ndx++) { - if (m_spec.get_column_type(spec_ndx) == col_type_BackLink) { - auto col_key = m_spec.get_key(spec_ndx); - // This function checks for a specific error in the m_keys array where the - // backlink column keys are wrong. It can be detected by trying to find the - // corresponding origin table. If the error exists some of the results will - // give null TableKeys back. - if (!get_opposite_table_key(col_key)) - return false; - auto t = get_opposite_table(col_key); - auto c = get_opposite_column(col_key); - if (!t->valid_column(c)) - return false; - if (t->get_opposite_column(c) != col_key) { - t->set_opposite_column(c, get_key(), col_key); - } - } - } - return true; - }; - - if (!check()) { - m_spec.fix_column_keys(get_key()); - build_column_mapping(); - refresh_index_accessors(); - REALM_ASSERT_RELEASE(check()); - modified = true; - } - return modified; -} - -// Delete the indexes stored in the columns array and create corresponding -// entries in m_index_accessors array. This also has the effect that the columns -// array after this step does not have extra entries for certain columns -void Table::migrate_indexes(ColKey pk_col_key) -{ - if (ref_type top_ref = m_top.get_as_ref(top_position_for_columns)) { - Array col_refs(m_alloc); - col_refs.set_parent(&m_top, top_position_for_columns); - col_refs.init_from_ref(top_ref); - auto col_count = m_spec.get_column_count(); - size_t col_ndx = 0; - - // If col_refs.size() equals col_count, there are no indexes to migrate - while (col_ndx < col_count && col_refs.size() > col_count) { - if (m_spec.get_column_attr(col_ndx).test(col_attr_Indexed) && !m_index_refs.get(col_ndx)) { - // Simply delete entry. This will have the effect that we will not have to take - // extra entries into account - auto old_index_ref = to_ref(col_refs.get(col_ndx + 1)); - col_refs.erase(col_ndx + 1); - if (old_index_ref) { - // It should not be possible for old_index_ref to be 0, but we have seen some error - // reports on freeing a null ref, so just to be sure ... - Array::destroy_deep(old_index_ref, m_alloc); - } - - // Primary key columns does not need an index - if (m_leaf_ndx2colkey[col_ndx] != pk_col_key) { - // Otherwise create new index. Will be updated when objects are created - m_index_accessors[col_ndx] = std::make_unique( - ClusterColumn(&m_clusters, m_spec.get_key(col_ndx), IndexType::General), - get_alloc()); // Throws - auto index = m_index_accessors[col_ndx].get(); - index->set_parent(&m_index_refs, col_ndx); - m_index_refs.set(col_ndx, index->get_ref()); - } - } - col_ndx++; - }; - } -} - -// Move information held in the subspec area into the structures managed by Table -// This is information about origin/target tables in relation to links -// This information is now held in "opposite" arrays directly in Table structure -// At the same time the backlink columns are destroyed -// If there is no subspec, this stage is done -void Table::migrate_subspec() -{ - if (!m_spec.has_subspec()) - return; - - ref_type top_ref = m_top.get_as_ref(top_position_for_columns); - Array col_refs(m_alloc); - col_refs.set_parent(&m_top, top_position_for_columns); - col_refs.init_from_ref(top_ref); - Group* group = get_parent_group(); - - for (size_t col_ndx = 0; col_ndx < m_spec.get_column_count(); col_ndx++) { - ColumnType col_type = m_spec.get_column_type(col_ndx); - - if (is_link_type(col_type)) { - auto target_key = m_spec.get_opposite_link_table_key(col_ndx); - auto target_table = group->get_table(target_key); - Spec& target_spec = _impl::TableFriend::get_spec(*target_table); - // The target table spec may already be migrated. - // If it has, the new functions should be used. - ColKey backlink_col_key = target_spec.has_subspec() - ? target_spec.find_backlink_column(m_key, col_ndx) - : target_table->find_opposite_column(m_spec.get_key(col_ndx)); - REALM_ASSERT(backlink_col_key.get_type() == col_type_BackLink); - if (m_opposite_table.get(col_ndx) != target_key.value) { - m_opposite_table.set(col_ndx, target_key.value); - } - if (m_opposite_column.get(col_ndx) != backlink_col_key.value) { - m_opposite_column.set(col_ndx, backlink_col_key.value); - } - } - else if (col_type == col_type_BackLink) { - auto origin_key = m_spec.get_opposite_link_table_key(col_ndx); - size_t origin_col_ndx = m_spec.get_origin_column_ndx(col_ndx); - auto origin_table = group->get_table(origin_key); - Spec& origin_spec = _impl::TableFriend::get_spec(*origin_table); - ColKey origin_col_key = origin_spec.get_key(origin_col_ndx); - REALM_ASSERT(is_link_type(origin_col_key.get_type())); - if (m_opposite_table.get(col_ndx) != origin_key.value) { - m_opposite_table.set(col_ndx, origin_key.value); - } - if (m_opposite_column.get(col_ndx) != origin_col_key.value) { - m_opposite_column.set(col_ndx, origin_col_key.value); - } - } - }; - m_spec.destroy_subspec(); -} - -namespace { - -class LegacyStringColumn : public BPlusTree { -public: - LegacyStringColumn(Allocator& alloc, Spec* spec, size_t col_ndx, bool nullable) - : BPlusTree(alloc) - , m_spec(spec) - , m_col_ndx(col_ndx) - , m_nullable(nullable) - { - } - - std::unique_ptr init_leaf_node(ref_type ref) override - { - auto leaf = std::make_unique(this); - leaf->ArrayString::set_spec(m_spec, m_col_ndx); - leaf->set_nullability(m_nullable); - leaf->init_from_ref(ref); - return leaf; - } - - StringData get_legacy(size_t n) const - { - if (m_cached_leaf_begin <= n && n < m_cached_leaf_end) { - return m_leaf_cache.get_legacy(n - m_cached_leaf_begin); - } - else { - StringData value; - - auto func = [&value](BPlusTreeNode* node, size_t ndx) { - auto leaf = static_cast(node); - value = leaf->get_legacy(ndx); - }; - - m_root->bptree_access(n, func); - - return value; - } - } - -private: - Spec* m_spec; - size_t m_col_ndx; - bool m_nullable; -}; - -// We need an accessor that can read old Timestamp columns. -// The new BPlusTree uses a different layout -class LegacyTS : private Array { -public: - explicit LegacyTS(Allocator& allocator) - : Array(allocator) - , m_seconds(allocator) - , m_nanoseconds(allocator) - { - m_seconds.set_parent(this, 0); - m_nanoseconds.set_parent(this, 1); - } - - using Array::set_parent; - - void init_from_parent() - { - Array::init_from_parent(); - m_seconds.init_from_parent(); - m_nanoseconds.init_from_parent(); - } - - size_t size() const - { - return m_seconds.size(); - } - - Timestamp get(size_t ndx) const - { - util::Optional seconds = m_seconds.get(ndx); - return seconds ? Timestamp(*seconds, int32_t(m_nanoseconds.get(ndx))) : Timestamp{}; - } - -private: - BPlusTree> m_seconds; - BPlusTree m_nanoseconds; -}; - -// Function that can retrieve a single value from the old columns -Mixed get_val_from_column(size_t ndx, ColumnType col_type, bool nullable, BPlusTreeBase* accessor) -{ - switch (col_type) { - case col_type_Int: - if (nullable) { - auto val = static_cast>*>(accessor)->get(ndx); - return Mixed{val}; - } - else { - return Mixed{static_cast*>(accessor)->get(ndx)}; - } - case col_type_Bool: - if (nullable) { - auto val = static_cast>*>(accessor)->get(ndx); - return val ? Mixed{bool(*val)} : Mixed{}; - } - else { - return Mixed{bool(static_cast*>(accessor)->get(ndx))}; - } - case col_type_Float: - return Mixed{static_cast*>(accessor)->get(ndx)}; - case col_type_Double: - return Mixed{static_cast*>(accessor)->get(ndx)}; - case col_type_String: { - auto str = static_cast(accessor)->get_legacy(ndx); - // This is a workaround for a bug where the length could be -1 - // Seen when upgrading very old file. - if (str.size() == size_t(-1)) { - return Mixed(""); - } - return Mixed{str}; - } - case col_type_Binary: - return Mixed{static_cast*>(accessor)->get(ndx)}; - default: - REALM_UNREACHABLE(); - } -} - -template -void copy_list(ref_type sub_table_ref, Obj& obj, ColKey col, Allocator& alloc) -{ - if (sub_table_ref) { - // Actual list is in the columns array position 0 - Array cols(alloc); - cols.init_from_ref(sub_table_ref); - ref_type list_ref = cols.get_as_ref(0); - BPlusTree from_list(alloc); - from_list.init_from_ref(list_ref); - size_t list_size = from_list.size(); - auto l = obj.get_list(col); - for (size_t j = 0; j < list_size; j++) { - l.add(from_list.get(j)); - } - } -} - -template <> -void copy_list>(ref_type sub_table_ref, Obj& obj, ColKey col, Allocator& alloc) -{ - if (sub_table_ref) { - // Actual list is in the columns array position 0 - Array cols(alloc); - cols.init_from_ref(sub_table_ref); - BPlusTree> from_list(alloc); - from_list.set_parent(&cols, 0); - from_list.init_from_parent(); - size_t list_size = from_list.size(); - auto l = obj.get_list>(col); - for (size_t j = 0; j < list_size; j++) { - util::Optional val; - auto int_val = from_list.get(j); - if (int_val) { - val = (*int_val != 0); - } - l.add(val); - } - } -} - -template <> -void copy_list(ref_type sub_table_ref, Obj& obj, ColKey col, Allocator& alloc) -{ - if (sub_table_ref) { - // Actual list is in the columns array position 0 - bool nullable = col.get_attrs().test(col_attr_Nullable); - Array cols(alloc); - cols.init_from_ref(sub_table_ref); - LegacyStringColumn from_list(alloc, nullptr, 0, nullable); // List of strings cannot be enumerated - from_list.set_parent(&cols, 0); - from_list.init_from_parent(); - size_t list_size = from_list.size(); - auto l = obj.get_list(col); - for (size_t j = 0; j < list_size; j++) { - l.add(from_list.get_legacy(j)); - } - } -} - -template <> -void copy_list(ref_type sub_table_ref, Obj& obj, ColKey col, Allocator& alloc) -{ - if (sub_table_ref) { - // Actual list is in the columns array position 0 - Array cols(alloc); - cols.init_from_ref(sub_table_ref); - LegacyTS from_list(alloc); - from_list.set_parent(&cols, 0); - from_list.init_from_parent(); - size_t list_size = from_list.size(); - auto l = obj.get_list(col); - for (size_t j = 0; j < list_size; j++) { - l.add(from_list.get(j)); - } - } -} - -} // namespace - -void Table::create_columns() -{ - size_t cnt; - auto get_column_cnt = [&cnt](const Cluster* cluster) { - cnt = cluster->nb_columns(); - return IteratorControl::Stop; - }; - traverse_clusters(get_column_cnt); - - size_t column_count = m_spec.get_column_count(); - if (cnt != column_count) { - for (size_t col_ndx = 0; col_ndx < column_count; col_ndx++) { - m_clusters.insert_column(m_spec.get_key(col_ndx)); - } - } -} - -bool Table::migrate_objects() -{ - size_t nb_public_columns = m_spec.get_public_column_count(); - size_t nb_columns = m_spec.get_column_count(); - if (!nb_columns) { - // No columns - this means no objects - return true; - } - ref_type top_ref = m_top.get_as_ref(top_position_for_columns); - if (!top_ref) { - // Has already been done - return true; - } - Array col_refs(m_alloc); - col_refs.set_parent(&m_top, top_position_for_columns); - col_refs.init_from_ref(top_ref); - - /************************ Create column accessors ************************/ - - std::map> column_accessors; - std::map> ts_accessors; - std::map>> list_accessors; - std::vector cols_to_destroy; - bool has_link_columns = false; - - // helper function to determine the number of objects in the table - size_t number_of_objects = (nb_columns == 0) ? 0 : size_t(-1); - auto update_size = [&number_of_objects](size_t s) { - if (number_of_objects == size_t(-1)) { - number_of_objects = s; - } - else { - REALM_ASSERT(s == number_of_objects); - } - }; - - for (size_t col_ndx = 0; col_ndx < nb_columns; col_ndx++) { - if (col_ndx < nb_public_columns && m_spec.get_column_name(col_ndx) == "!ROW_INDEX") { - // If this column has been added, we can break here - break; - } - - ColKey col_key = m_spec.get_key(col_ndx); - ColumnAttrMask attr = m_spec.get_column_attr(col_ndx); - ColumnType col_type = m_spec.get_column_type(col_ndx); - bool nullable = attr.test(col_attr_Nullable); - std::unique_ptr acc; - std::unique_ptr ts_acc; - std::unique_ptr> list_acc; - - if (!(col_ndx < col_refs.size())) { - throw std::runtime_error( - util::format("Objects in '%1' corrupted by previous upgrade attempt", get_name())); - } - - if (!col_refs.get(col_ndx)) { - // This column has been migrated - continue; - } - - if (attr.test(col_attr_List) && col_type != col_type_LinkList) { - list_acc = std::make_unique>(m_alloc); - } - else { - switch (col_type) { - case col_type_Int: - case col_type_Bool: - if (nullable) { - acc = std::make_unique>>(m_alloc); - } - else { - acc = std::make_unique>(m_alloc); - } - break; - case col_type_Float: - acc = std::make_unique>(m_alloc); - break; - case col_type_Double: - acc = std::make_unique>(m_alloc); - break; - case col_type_String: - acc = std::make_unique(m_alloc, &m_spec, col_ndx, nullable); - break; - case col_type_Binary: - acc = std::make_unique>(m_alloc); - break; - case col_type_Timestamp: { - ts_acc = std::make_unique(m_alloc); - break; - } - case col_type_Link: - case col_type_LinkList: { - BPlusTree arr(m_alloc); - arr.set_parent(&col_refs, col_ndx); - arr.init_from_parent(); - update_size(arr.size()); - has_link_columns = true; - break; - } - case col_type_BackLink: { - BPlusTree arr(m_alloc); - arr.set_parent(&col_refs, col_ndx); - arr.init_from_parent(); - update_size(arr.size()); - cols_to_destroy.push_back(col_ndx); - break; - } - default: - break; - } - } - - if (acc) { - acc->set_parent(&col_refs, col_ndx); - acc->init_from_parent(); - update_size(acc->size()); - column_accessors.emplace(col_key, std::move(acc)); - cols_to_destroy.push_back(col_ndx); - } - if (ts_acc) { - ts_acc->set_parent(&col_refs, col_ndx); - ts_acc->init_from_parent(); - update_size(ts_acc->size()); - ts_accessors.emplace(col_key, std::move(ts_acc)); - cols_to_destroy.push_back(col_ndx); - } - if (list_acc) { - list_acc->set_parent(&col_refs, col_ndx); - list_acc->init_from_parent(); - update_size(list_acc->size()); - list_accessors.emplace(col_key, std::move(list_acc)); - cols_to_destroy.push_back(col_ndx); - } - } - - REALM_ASSERT(number_of_objects != size_t(-1)); - - if (m_clusters.size() == number_of_objects) { - // We have migrated all objects - return !has_link_columns; - } - - // !OID column must not be present. Such columns are only present in syncked - // realms, which we cannot upgrade. - REALM_ASSERT(nb_public_columns == 0 || m_spec.get_column_name(0) != "!OID"); - - /*************************** Create objects ******************************/ - - for (size_t row_ndx = 0; row_ndx < number_of_objects; row_ndx++) { - // Build a vector of values obtained from the old columns - FieldValues init_values; - for (auto& it : column_accessors) { - auto col_key = it.first; - auto col_type = col_key.get_type(); - auto nullable = col_key.get_attrs().test(col_attr_Nullable); - auto val = get_val_from_column(row_ndx, col_type, nullable, it.second.get()); - init_values.insert(col_key, val); - } - for (auto& it : ts_accessors) { - init_values.insert(it.first, Mixed(it.second->get(row_ndx))); - } - // Create object with the initial values - Obj obj = m_clusters.insert(ObjKey(row_ndx), init_values); - - // Then update possible list types - for (auto& it : list_accessors) { - switch (it.first.get_type()) { - case col_type_Int: { - if (it.first.get_attrs().test(col_attr_Nullable)) { - copy_list>(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - } - else { - copy_list(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - } - break; - } - case col_type_Bool: - if (it.first.get_attrs().test(col_attr_Nullable)) { - copy_list>(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - } - else { - copy_list(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - } - break; - case col_type_Float: - copy_list(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - break; - case col_type_Double: - copy_list(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - break; - case col_type_String: - copy_list(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - break; - case col_type_Binary: - copy_list(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - break; - case col_type_Timestamp: { - copy_list(to_ref(it.second->get(row_ndx)), obj, it.first, m_alloc); - break; - } - default: - break; - } - } - } - - // Destroy values in the old columns that has been copied. - // This frees up space in the file - for (auto ndx : cols_to_destroy) { - Array::destroy_deep(to_ref(col_refs.get(ndx)), m_alloc); - col_refs.set(ndx, 0); - } - - // We need to be sure that the stored 'next sequence number' is bigger than - // the biggest ObjKey currently used. - this->set_sequence_number(uint64_t(number_of_objects)); - -#if 0 - if (fastrand(100) < 20) { - throw util::runtime_error("Upgrade interrupted"); - } -#endif - return !has_link_columns; -} - -void Table::migrate_links() -{ - ref_type top_ref = m_top.get_as_ref(top_position_for_columns); - if (!top_ref) { - // All objects migrated - return; - } - - Array col_refs(m_alloc); - col_refs.set_parent(&m_top, top_position_for_columns); - col_refs.init_from_ref(top_ref); - - // Cache column accessors and other information - size_t nb_columns = m_spec.get_public_column_count(); - std::vector>> link_column_accessors(nb_columns); - std::vector col_keys(nb_columns); - std::vector col_types(nb_columns); - std::vector target_tables(nb_columns); - std::vector opposite_orig_row_ndx_col(nb_columns); - for (size_t col_ndx = 0; col_ndx < nb_columns; col_ndx++) { - ColumnType col_type = m_spec.get_column_type(col_ndx); - - if (is_link_type(col_type)) { - link_column_accessors[col_ndx] = std::make_unique>(m_alloc); - link_column_accessors[col_ndx]->set_parent(&col_refs, col_ndx); - link_column_accessors[col_ndx]->init_from_parent(); - col_keys[col_ndx] = m_spec.get_key(col_ndx); - col_types[col_ndx] = col_type; - target_tables[col_ndx] = get_opposite_table(col_keys[col_ndx]).unchecked_ptr(); - opposite_orig_row_ndx_col[col_ndx] = target_tables[col_ndx]->get_column_key("!ROW_INDEX"); - } - } - - auto orig_row_ndx_col_key = get_column_key("!ROW_INDEX"); - for (auto obj : *this) { - for (size_t col_ndx = 0; col_ndx < nb_columns; col_ndx++) { - if (col_keys[col_ndx]) { - // If no !ROW_INDEX column is found, the original row index number is - // equal to the ObjKey value - size_t orig_row_ndx = - size_t(orig_row_ndx_col_key ? obj.get(orig_row_ndx_col_key) : obj.get_key().value); - // Get original link value - int64_t link_val = link_column_accessors[col_ndx]->get(orig_row_ndx); - - Table* target_table = target_tables[col_ndx]; - ColKey search_col = opposite_orig_row_ndx_col[col_ndx]; - auto get_target_key = [target_table, search_col](int64_t orig_link_val) -> ObjKey { - if (search_col) - return target_table->find_first_int(search_col, orig_link_val); - else - return ObjKey(orig_link_val); - }; - - if (link_val) { - if (col_types[col_ndx] == col_type_Link) { - obj.set(col_keys[col_ndx], get_target_key(link_val - 1)); - } - else { - auto ll = obj.get_linklist(col_keys[col_ndx]); - BPlusTree links(m_alloc); - links.init_from_ref(ref_type(link_val)); - size_t nb_links = links.size(); - for (size_t j = 0; j < nb_links; j++) { - ll.add(get_target_key(links.get(j))); - } - } - } - } - } - } -} - -void Table::finalize_migration(ColKey pk_col_key) -{ - if (ref_type ref = m_top.get_as_ref(top_position_for_columns)) { - Array::destroy_deep(ref, m_alloc); - m_top.set(top_position_for_columns, 0); - } - - if (auto orig_row_ndx_col = get_column_key("!ROW_INDEX")) { - remove_column(orig_row_ndx_col); - } - - if (auto oid_col = get_column_key("!OID")) { - remove_column(oid_col); - } - - REALM_ASSERT_RELEASE(!pk_col_key || valid_column(pk_col_key)); - do_set_primary_key_column(pk_col_key); -} void Table::migrate_sets_and_dictionaries() { diff --git a/src/realm/table.hpp b/src/realm/table.hpp index b622adbcc37..f5ea191f6f4 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -717,14 +717,6 @@ class Table { void clear_indexes(); // Migration support - void migrate_column_info(); - bool verify_column_keys(); - void migrate_indexes(ColKey pk_col_key); - void migrate_subspec(); - void create_columns(); - bool migrate_objects(); // Returns true if there are no links to migrate - void migrate_links(); - void finalize_migration(ColKey pk_col_key); void migrate_sets_and_dictionaries(); /// Disable copying assignment. diff --git a/src/realm/transaction.cpp b/src/realm/transaction.cpp index a5ea824ccba..26bc5e46ce6 100644 --- a/src/realm/transaction.cpp +++ b/src/realm/transaction.cpp @@ -459,7 +459,7 @@ void Transaction::upgrade_file_format(int target_file_format_version) // Be sure to revisit the following upgrade logic when a new file format // version is introduced. The following assert attempt to help you not // forget it. - REALM_ASSERT_EX(target_file_format_version == 23, target_file_format_version); + REALM_ASSERT_EX(target_file_format_version == 24, target_file_format_version); // DB::do_open() must ensure that only supported version are allowed. // It does that by asking backup if the current file format version is @@ -469,125 +469,6 @@ void Transaction::upgrade_file_format(int target_file_format_version) int current_file_format_version = get_file_format_version(); REALM_ASSERT(current_file_format_version < target_file_format_version); - // Upgrade from version prior to 7 (new history schema version in top array) - if (current_file_format_version <= 6 && target_file_format_version >= 7) { - // If top array size is 9, then add the missing 10th element containing - // the history schema version. - std::size_t top_size = m_top.size(); - REALM_ASSERT(top_size <= 9); - if (top_size == 9) { - int initial_history_schema_version = 0; - m_top.add(initial_history_schema_version); // Throws - } - set_file_format_version(7); - commit_and_continue_writing(); - } - - // Upgrade from version prior to 10 (Cluster based db) - if (current_file_format_version <= 9 && target_file_format_version >= 10) { - DisableReplication disable_replication(*this); - - std::vector table_accessors; - TableRef pk_table; - TableRef progress_info; - ColKey col_objects; - ColKey col_links; - std::map pk_cols; - - // Use table lookup by name. The table keys are not generated yet - for (size_t t = 0; t < m_table_names.size(); t++) { - StringData name = m_table_names.get(t); - // In file format version 9 files, all names represent existing tables. - auto table = get_table(name); - if (name == "pk") { - pk_table = table; - } - else if (name == "!UPDATE_PROGRESS") { - progress_info = table; - } - else { - table_accessors.push_back(table); - } - } - - if (!progress_info) { - // This is the first time. Prepare for moving objects in one go. - progress_info = this->add_table_with_primary_key("!UPDATE_PROGRESS", type_String, "table_name"); - col_objects = progress_info->add_column(type_Bool, "objects_migrated"); - col_links = progress_info->add_column(type_Bool, "links_migrated"); - - - for (auto k : table_accessors) { - k->migrate_column_info(); - } - - if (pk_table) { - pk_table->migrate_column_info(); - pk_table->migrate_indexes(ColKey()); - pk_table->create_columns(); - pk_table->migrate_objects(); - pk_cols = get_primary_key_columns_from_pk_table(pk_table); - } - - for (auto k : table_accessors) { - k->migrate_indexes(pk_cols[k]); - } - for (auto k : table_accessors) { - k->migrate_subspec(); - } - for (auto k : table_accessors) { - k->create_columns(); - } - commit_and_continue_writing(); - } - else { - if (pk_table) { - pk_cols = get_primary_key_columns_from_pk_table(pk_table); - } - col_objects = progress_info->get_column_key("objects_migrated"); - col_links = progress_info->get_column_key("links_migrated"); - } - - bool updates = false; - for (auto k : table_accessors) { - if (k->verify_column_keys()) { - updates = true; - } - } - if (updates) { - commit_and_continue_writing(); - } - - // Migrate objects - for (auto k : table_accessors) { - auto progress_status = progress_info->create_object_with_primary_key(k->get_name()); - if (!progress_status.get(col_objects)) { - bool no_links = k->migrate_objects(); - progress_status.set(col_objects, true); - progress_status.set(col_links, no_links); - commit_and_continue_writing(); - } - } - for (auto k : table_accessors) { - auto progress_status = progress_info->create_object_with_primary_key(k->get_name()); - if (!progress_status.get(col_links)) { - k->migrate_links(); - progress_status.set(col_links, true); - commit_and_continue_writing(); - } - } - - // Final cleanup - for (auto k : table_accessors) { - k->finalize_migration(pk_cols[k]); - } - - if (pk_table) { - remove_table("pk"); - } - remove_table(progress_info->get_key()); - } - // Ensure we have search index on all primary key columns. auto table_keys = get_table_keys(); if (current_file_format_version < 22) { diff --git a/test/object-store/backup.cpp b/test/object-store/backup.cpp index 52d1c5be089..91a1ed36b8e 100644 --- a/test/object-store/backup.cpp +++ b/test/object-store/backup.cpp @@ -69,7 +69,7 @@ TEST_CASE("Automated backup") { util::File::copy(copy_from_file_name, config.path); REQUIRE(util::File::exists(config.path)); // backup name must reflect version of old realm file (which is v6) - std::string backup_path = test_util::get_test_path_prefix() + "test_backup.v6.backup.realm"; + std::string backup_path = test_util::get_test_path_prefix() + "test_backup.v20.backup.realm"; std::string backup_log = test_util::get_test_path_prefix() + "test_backup.realm.backup-log"; util::File::try_remove(backup_path); util::File::try_remove(backup_log); diff --git a/test/object-store/test_backup-olden-and-golden.realm b/test/object-store/test_backup-olden-and-golden.realm index 352f0894db4..25112f1653a 100644 Binary files a/test/object-store/test_backup-olden-and-golden.realm and b/test/object-store/test_backup-olden-and-golden.realm differ diff --git a/test/test_upgrade_colkey_error.realm b/test/test_upgrade_colkey_error.realm deleted file mode 100644 index c9c0fdf4c14..00000000000 Binary files a/test/test_upgrade_colkey_error.realm and /dev/null differ diff --git a/test/test_upgrade_database.cpp b/test/test_upgrade_database.cpp index ac3df5c1084..5b0eb905e78 100644 --- a/test/test_upgrade_database.cpp +++ b/test/test_upgrade_database.cpp @@ -195,710 +195,6 @@ NONCONCURRENT_TEST(Upgrade_DatabaseWithUnsupportedNewFileFormat) compare_files(test_context, path, path_2); } -// Open an existing database-file-format-version 5 file and -// check that it automatically upgrades to version 6. -TEST_IF(Upgrade_Database_5_6_StringIndex, REALM_MAX_BPNODE_SIZE == 1000) -{ - std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + - util::to_string(REALM_MAX_BPNODE_SIZE) + "_5_to_6_stringindex.realm"; - - // use a common prefix which will not cause a stack overflow but is larger - // than StringIndex::s_max_offset - const int common_prefix_length = 500; - std::string std_base2(common_prefix_length, 'a'); - std::string std_base2_b = std_base2 + "b"; - std::string std_base2_c = std_base2 + "c"; - StringData base2(std_base2); - StringData base2_b(std_base2_b); - StringData base2_c(std_base2_c); - - -#if TEST_READ_UPGRADE_MODE - - // Automatic upgrade from SharedGroup - { - CHECK_OR_RETURN(File::exists(path)); - SHARED_GROUP_TEST_PATH(temp_copy); - - // Make a copy of the database so that we keep the original file intact and unmodified - File::copy(path, temp_copy); - - // Constructing this SharedGroup will trigger an upgrade - // for all tables because the file is in version 4 - auto sg = DB::create(temp_copy); - - WriteTransaction wt(sg); - TableRef t = wt.get_table("t1"); - TableRef t2 = wt.get_table("t2"); - - auto int_ndx = t->get_column_key("int"); - auto bool_ndx = t->get_column_key("bool"); - auto str_ndx = t->get_column_key("string"); - auto ts_ndx = t->get_column_key("timestamp"); - - auto null_int_ndx = t->get_column_key("nullable int"); - auto null_bool_ndx = t->get_column_key("nullable bool"); - auto null_str_ndx = t->get_column_key("nullable string"); - auto null_ts_ndx = t->get_column_key("nullable timestamp"); - - size_t num_rows = 6; - - CHECK_EQUAL(t->size(), num_rows); - CHECK(t->has_search_index(int_ndx)); - CHECK(t->has_search_index(bool_ndx)); - CHECK(t->has_search_index(str_ndx)); - CHECK(t->has_search_index(ts_ndx)); - - CHECK(t->has_search_index(null_int_ndx)); - CHECK(t->has_search_index(null_bool_ndx)); - CHECK(t->has_search_index(null_str_ndx)); - CHECK(t->has_search_index(null_ts_ndx)); - - CHECK_EQUAL(t->find_first_string(str_ndx, base2_b), ObjKey(0)); - CHECK_EQUAL(t->find_first_string(str_ndx, base2_c), ObjKey(1)); - CHECK_EQUAL(t->find_first_string(str_ndx, base2), ObjKey(4)); - // CHECK_EQUAL(t->get_distinct_view(str_ndx).size(), 4); - CHECK_EQUAL(t->size(), 6); - - CHECK_EQUAL(t->find_first_string(null_str_ndx, base2_b), ObjKey(0)); - CHECK_EQUAL(t->find_first_string(null_str_ndx, base2_c), ObjKey(1)); - CHECK_EQUAL(t->find_first_string(null_str_ndx, base2), ObjKey(4)); - // CHECK_EQUAL(t->get_distinct_view(null_str_ndx).size(), 4); - - // If the StringIndexes were not updated we couldn't do this - // on a format 5 file and find it again. - std::string std_base2_d = std_base2 + "d"; - StringData base2_d(std_base2_d); - auto obj = t->create_object().set(str_ndx, base2_d); - CHECK_EQUAL(t->find_first_string(str_ndx, base2_d), ObjKey(6)); - obj.set(null_str_ndx, base2_d); - CHECK_EQUAL(t->find_first_string(null_str_ndx, base2_d), ObjKey(6)); - - // And if the indexes were using the old format, adding a long - // prefix string would cause a stack overflow. - std::string big_base(90000, 'a'); - std::string big_base_b = big_base + "b"; - std::string big_base_c = big_base + "c"; - StringData b(big_base_b); - StringData c(big_base_c); - t->create_object().set(str_ndx, b).set(null_str_ndx, b); - t->create_object().set(str_ndx, c).set(null_str_ndx, c); - - t->verify(); - t2->verify(); - } - -#else // test write mode -#ifndef REALM_CLUSTER_IF - // NOTE: This code must be executed from an old file-format-version 5 - // core in order to create a file-format-version 5 test file! - char leafsize[20]; - sprintf(leafsize, "%d", REALM_MAX_BPNODE_SIZE); - File::try_remove(path); - - Group g; - TableRef t = g.add_table("t1"); - TableRef t2 = g.add_table("t2"); - - size_t int_ndx = t->add_column(type_Int, "int"); - size_t bool_ndx = t->add_column(type_Bool, "bool"); - t->add_column(type_Float, "float"); - t->add_column(type_Double, "double"); - size_t str_ndx = t->add_column(type_String, "string"); - t->add_column(type_Binary, "binary"); - size_t ts_ndx = t->add_column(type_Timestamp, "timestamp"); - t->add_column(type_Table, "table"); - t->add_column(type_Mixed, "mixed"); - t->add_column_link(type_Link, "link", *t2); - t->add_column_link(type_LinkList, "linklist", *t2); - - size_t null_int_ndx = t->add_column(type_Int, "nullable int", true); - size_t null_bool_ndx = t->add_column(type_Bool, "nullable bool", true); - size_t null_str_ndx = t->add_column(type_String, "nullable string", true); - size_t null_ts_ndx = t->add_column(type_Timestamp, "nullable timestamp", true); - - t->add_search_index(bool_ndx); - t->add_search_index(int_ndx); - t->add_search_index(str_ndx); - t->add_search_index(ts_ndx); - t->add_search_index(null_bool_ndx); - t->add_search_index(null_int_ndx); - t->add_search_index(null_str_ndx); - t->add_search_index(null_ts_ndx); - - t->add_empty_row(6); - t->set_string(str_ndx, 0, base2_b); - t->set_string(str_ndx, 1, base2_c); - t->set_string(str_ndx, 2, "aaaaaaaaaa"); - t->set_string(str_ndx, 3, "aaaaaaaaaa"); - t->set_string(str_ndx, 4, base2); - t->set_string(str_ndx, 5, base2); - - t->set_string(null_str_ndx, 0, base2_b); - t->set_string(null_str_ndx, 1, base2_c); - t->set_string(null_str_ndx, 2, "aaaaaaaaaa"); - t->set_string(null_str_ndx, 3, "aaaaaaaaaa"); - t->set_string(null_str_ndx, 4, base2); - t->set_string(null_str_ndx, 5, base2); - - g.write(path); -#endif // REALM_CLUSTER_IF -#endif // TEST_READ_UPGRADE_MODE -} - - -TEST_IF(Upgrade_Database_6_7, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE == 1000) -{ - std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + - util::to_string(REALM_MAX_BPNODE_SIZE) + "_6_to_7.realm"; - -#if TEST_READ_UPGRADE_MODE - - // Automatic upgrade from SharedGroup - { - CHECK_OR_RETURN(File::exists(path)); - SHARED_GROUP_TEST_PATH(temp_copy); - - // Make a copy of the database so that we keep the original file intact and unmodified - File::copy(path, temp_copy); - - // Constructing this SharedGroup will trigger an upgrade - auto hist = make_in_realm_history(); - DBRef sg = DB::create(*hist, temp_copy); - - auto rt = sg->start_read(); - CHECK_EQUAL(rt->get_history_schema_version(), hist->get_history_schema_version()); - - ConstTableRef t = rt->get_table("table"); - auto col = t->get_column_key("value"); - CHECK(t); - CHECK_EQUAL(t->size(), 1); - CHECK_EQUAL(t->begin()->get(col), 123); - } -#else // test write mode -#ifndef REALM_CLUSTER_IF - // NOTE: This code must be executed from an old file-format-version 6 - // core in order to create a file-format-version 6 test file! - - Group g; - TableRef t = g.add_table("table"); - size_t col = t->add_column(type_Int, "value"); - size_t row = t->add_empty_row(); - t->set_int(col, row, 123); - g.write(path); -#endif // REALM_CLUSTER_IF -#endif // TEST_READ_UPGRADE_MODE -} - -TEST_IF(Upgrade_Database_7_8, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE == 1000) -{ - std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + - util::to_string(REALM_MAX_BPNODE_SIZE) + "_7_to_8.realm"; - -#if TEST_READ_UPGRADE_MODE - - // Automatic upgrade from SharedGroup - { - CHECK_OR_RETURN(File::exists(path)); - SHARED_GROUP_TEST_PATH(temp_copy); - - // Make a copy of the database so that we keep the original file intact and unmodified - File::copy(path, temp_copy); - - // Constructing this SharedGroup will trigger an upgrade - auto hist = make_in_realm_history(); - DBRef sg = DB::create(*hist, temp_copy); - - auto rt = sg->start_read(); - CHECK_EQUAL(rt->get_history_schema_version(), hist->get_history_schema_version()); - - ConstTableRef t = rt->get_table("table"); - auto col = t->get_column_key("value"); - CHECK(t); - CHECK_EQUAL(t->size(), 1); - CHECK_EQUAL(t->begin()->get(col), 123); - } -#else // test write mode -#ifndef REALM_CLUSTER_IF - // NOTE: This code must be executed from an old file-format-version 7 - // core in order to create a file-format-version 7 test file! - - Group g; - TableRef t = g.add_table("table"); - size_t col = t->add_column(type_Int, "value"); - size_t row = t->add_empty_row(); - t->set_int(col, row, 123); - g.write(path); -#endif // REALM_CLUSTER_IF -#endif // TEST_READ_UPGRADE_MODE -} - - -TEST_IF(Upgrade_Database_8_9, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE == 1000) -{ - std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + - util::to_string(REALM_MAX_BPNODE_SIZE) + "_8_to_9.realm"; - std::string validation_str = "test string"; -#if TEST_READ_UPGRADE_MODE - - // Automatic upgrade from SharedGroup - { - CHECK_OR_RETURN(File::exists(path)); - SHARED_GROUP_TEST_PATH(temp_copy); - - // Make a copy of the database so that we keep the original file intact and unmodified - File::copy(path, temp_copy); - - // Constructing this SharedGroup will trigger an upgrade - auto hist = make_in_realm_history(); - DBRef sg = DB::create(*hist, temp_copy); - - auto rt = sg->start_read(); - CHECK_EQUAL(rt->get_history_schema_version(), hist->get_history_schema_version()); - - ConstTableRef t = rt->get_table("table"); - auto col_int = t->get_column_key("value"); - auto col_str = t->get_column_key("str_col"); - CHECK(t); - CHECK_EQUAL(t->size(), 1); - CHECK_EQUAL(t->begin()->get(col_int), 123); - CHECK_EQUAL(t->begin()->get(col_str), validation_str); - } -#else // test write mode -#ifndef REALM_CLUSTER_IF - // NOTE: This code must be executed from an old file-format-version 8 - // core in order to create a file-format-version 8 test file! - - Group g; - TableRef t = g.add_table("table"); - size_t col = t->add_column(type_Int, "value"); - size_t str_col = t->add_column(type_String, "str_col", true); - t->add_search_index(str_col); - size_t row = t->add_empty_row(); - t->set_int(col, row, 123); - t->set_string(str_col, row, validation_str); - g.write(path); -#endif // REALM_CLUSTER_IF -#endif // TEST_READ_UPGRADE_MODE -} - -TEST(Upgrade_Database_6_10) -{ - std::string path = test_util::get_test_resource_path() + "test_upgrade_database_6.realm"; - CHECK_OR_RETURN(File::exists(path)); - - SHARED_GROUP_TEST_PATH(temp_copy); - - // Make a copy of the database so that we keep the original file intact and unmodified - File::copy(path, temp_copy); - auto hist = make_in_realm_history(); - DBRef sg = DB::create(*hist, temp_copy); - ReadTransaction rt(sg); - auto t = rt.get_table("table"); - CHECK(t); -} - -namespace { -constexpr bool generate_json = false; -} - -TEST(Upgrade_Database_9_10_with_pk_table) -{ - /* File has this content: - { - "pk":[ - {"_key":0,"pk_table":"link origin","pk_property":"pk"} - {"_key":1,"pk_table":"object","pk_property":"pk"} - ], - "metadata":[ - {"_key":0,"version":0} - ], - "class_dog":[ - ], - "class_link origin":[ - {"_key":0,"pk":5,"object":null,"array":{"table": "class_object", "keys": []}}, - {"_key":1,"pk":6,"object":{"table": "class_object", "key": 0},"array":{"table": "class_object", "keys": []}}, - {"_key":2,"pk":7,"object":null,"array":{"table": "class_object", "keys": [1,2]}} - ], - "class_object":[ - {"_key":0,"pk":"hello","value":7,"enum":"red","list":[""],"optional":null}, - {"_key":1,"pk":"world","value":35,"enum":"blue","list":[],"optional":null}, - {"_key":2,"pk":"goodbye","value":800,"enum":"red","list":[],"optional":-87} - ] - } - */ - - std::string path = test_util::get_test_resource_path() + "test_upgrade_database_9_to_10_pk_table.realm"; - SHARED_GROUP_TEST_PATH(temp_copy); - - // Make a copy of the database so that we keep the original file intact and unmodified - File::copy(path, temp_copy); - auto hist = make_in_realm_history(); - auto sg = DB::create(*hist, temp_copy); - ReadTransaction rt(sg); - rt.get_group().verify(); - CHECK_EQUAL(rt.get_group().size(), 4); - // rt.get_group().to_json(std::cout); - - ConstTableRef t_object = rt.get_table("class_object"); - ConstTableRef t_origin = rt.get_table("class_link origin"); - ConstTableRef t_dog = rt.get_table("class_dog"); - - auto pk_col = t_object->get_primary_key_column(); - CHECK(pk_col); - CHECK_EQUAL(t_object->get_column_name(pk_col), "pk"); - auto hello_key = t_object->find_first_string(pk_col, "hello"); - auto obj1 = t_object->get_object(hello_key); - CHECK_EQUAL(obj1.get("value"), 7); - auto enum_col_key = t_object->get_column_key("enum"); - CHECK(t_object->is_enumerated(enum_col_key)); - CHECK_EQUAL(obj1.get(enum_col_key), "red"); - auto list = obj1.get_list(t_object->get_column_key("list")); - CHECK_EQUAL(list.size(), 1); - CHECK_EQUAL(list.get(0), ""); - - pk_col = t_origin->get_primary_key_column(); - CHECK(pk_col); - CHECK_EQUAL(t_origin->get_column_name(pk_col), "pk"); - auto key_6 = t_origin->find_first_int(pk_col, 6); - auto obj2 = t_origin->get_object(key_6); - CHECK_EQUAL(obj2.get("object"), hello_key); - - pk_col = t_dog->get_primary_key_column(); - CHECK(pk_col); -} - -TEST_IF(Upgrade_Database_9_10, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE == 1000) -{ - size_t nb_rows = (REALM_MAX_BPNODE_SIZE == 4) ? 50 : 500; - size_t insert_pos = (REALM_MAX_BPNODE_SIZE == 4) ? 40 : 177; - - std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + - util::to_string(REALM_MAX_BPNODE_SIZE) + "_9_to_10.realm"; -#if TEST_READ_UPGRADE_MODE - CHECK_OR_RETURN(File::exists(path)); - - SHARED_GROUP_TEST_PATH(temp_copy); - - // Make a copy of the database so that we keep the original file intact and unmodified - File::copy(path, temp_copy); - auto hist = make_in_realm_history(); - - int iter = 2; - while (iter) { - int max_try = 10; - DBRef sg; - while (max_try--) { - try { - // Constructing this SharedGroup will trigger an upgrade first time around - sg = DB::create(*hist, temp_copy); - break; - } - catch (...) { - // Upgrade failed - try again - } - } - ReadTransaction rt(sg); - - ConstTableRef t = rt.get_table("table"); - ConstTableRef o = rt.get_table("other"); - ConstTableRef e = rt.get_table("empty"); - rt.get_group().verify(); - - CHECK(t); - CHECK(o); - CHECK_EQUAL(t->size(), nb_rows + 1); - CHECK_EQUAL(o->size(), 25); - CHECK_EQUAL(e->size(), 0); - - auto t_col_keys = t->get_column_keys(); - CHECK_EQUAL(t_col_keys.size(), 14); - auto o_col_keys = o->get_column_keys(); - CHECK_EQUAL(o_col_keys.size(), 2); - auto e_col_keys = e->get_column_keys(); - CHECK_EQUAL(e_col_keys.size(), 0); - - auto col_int = t_col_keys[0]; - auto col_int_null = t_col_keys[1]; - auto col_bool = t_col_keys[2]; - auto col_bool_null = t_col_keys[3]; - auto col_float = t_col_keys[4]; - auto col_double = t_col_keys[5]; - auto col_string = t_col_keys[6]; - auto col_string_i = t_col_keys[7]; - auto col_binary = t_col_keys[8]; - auto col_date = t_col_keys[9]; - auto col_link = t_col_keys[10]; - auto col_linklist = t_col_keys[11]; - auto col_int_list = t_col_keys[12]; - auto col_int_null_list = t_col_keys[13]; - auto col_val = o_col_keys[0]; - auto col_str_list = o_col_keys[1]; - - CHECK_EQUAL(t->get_column_name(col_int), "int"); - CHECK_EQUAL(t->get_column_name(col_int_null), "int_1"); - CHECK_EQUAL(t->get_column_name(col_bool), "col_2"); - CHECK_EQUAL(t->get_column_name(col_bool_null), "bool_null"); - CHECK_EQUAL(t->get_column_name(col_float), "float"); - CHECK_EQUAL(t->get_column_name(col_double), "double"); - CHECK_EQUAL(t->get_column_name(col_string), "string"); - CHECK_EQUAL(t->get_column_name(col_string_i), "string_i"); - CHECK_EQUAL(t->get_column_name(col_binary), "binary"); - CHECK_EQUAL(t->get_column_name(col_date), "date"); - CHECK_EQUAL(t->get_column_name(col_link), "link"); - CHECK_EQUAL(t->get_column_name(col_linklist), "linklist"); - CHECK_EQUAL(t->get_column_name(col_int_list), "integers"); - CHECK_EQUAL(o->get_column_name(col_val), "val"); - CHECK_EQUAL(o->get_column_name(col_str_list), "strings"); - - CHECK_EQUAL(t->get_column_type(col_int_null), type_Int); - CHECK_EQUAL(t->get_column_type(col_bool), type_Bool); - CHECK_EQUAL(t->get_column_type(col_bool_null), type_Bool); - CHECK_EQUAL(t->get_column_type(col_float), type_Float); - CHECK_EQUAL(t->get_column_type(col_double), type_Double); - CHECK_EQUAL(t->get_column_type(col_string), type_String); - CHECK_EQUAL(t->get_column_type(col_string_i), type_String); - CHECK_EQUAL(t->get_column_type(col_binary), type_Binary); - CHECK_EQUAL(t->get_column_type(col_date), type_Timestamp); - CHECK_EQUAL(t->get_column_type(col_link), type_Link); - CHECK_EQUAL(t->get_column_type(col_linklist), type_LinkList); - CHECK_EQUAL(t->get_column_type(col_int_list), type_Int); - CHECK(t->get_column_attr(col_int_list).test(col_attr_List)); - CHECK_EQUAL(t->get_column_type(col_int_null_list), type_Int); - CHECK(t->get_column_attr(col_int_null_list).test(col_attr_List)); - CHECK_EQUAL(o->get_column_type(col_val), type_Int); - CHECK_EQUAL(o->get_column_type(col_str_list), type_String); - CHECK(col_str_list.get_attrs().test(col_attr_List)); - - CHECK_EQUAL(t->is_nullable(col_int), false); - CHECK_EQUAL(t->is_nullable(col_int_null), true); - CHECK_EQUAL(t->is_nullable(col_bool), false); - CHECK_EQUAL(t->is_nullable(col_bool_null), true); - CHECK_EQUAL(t->is_nullable(col_float), false); - CHECK_EQUAL(t->is_nullable(col_double), false); - CHECK_EQUAL(t->is_nullable(col_string), false); - CHECK_EQUAL(t->is_nullable(col_string_i), true); - CHECK_EQUAL(t->is_nullable(col_binary), false); - CHECK_EQUAL(t->is_nullable(col_date), false); - CHECK_EQUAL(t->is_nullable(col_link), true); - CHECK_EQUAL(t->is_nullable(col_linklist), false); - CHECK_EQUAL(t->is_nullable(col_int_list), false); - CHECK_EQUAL(t->is_nullable(col_int_null_list), true); - - CHECK_EQUAL(t->has_search_index(col_string), false); - CHECK_EQUAL(t->has_search_index(col_string_i), true); - - const Obj obj0 = t->get_object(ObjKey(0)); - CHECK(obj0.is_null(col_int_null)); - CHECK(obj0.is_null(col_bool_null)); - - const Obj obj17 = t->get_object(ObjKey(17)); - const Obj obj18 = t->get_object(ObjKey(18)); - const Obj obj23 = t->get_object(ObjKey(23)); - const Obj obj27 = t->get_object(ObjKey(27)); - const Obj obj = t->get_object(ObjKey(insert_pos)); - - CHECK_EQUAL(obj17.get(col_int), 17); - CHECK_EQUAL(obj17.get>(col_int_null), 17); - CHECK_EQUAL(obj17.get(col_bool), false); - CHECK_EQUAL(obj17.get>(col_bool_null), false); - CHECK_EQUAL(obj17.get(col_float), 17 * 1.5f); - CHECK_EQUAL(obj17.get(col_double), 17 * 2.5); - CHECK_EQUAL(obj17.get(col_string), "This is a medium long string"); - CHECK_EQUAL(t->find_first(col_string_i, StringData("This is a somewhat longer string17")), obj17.get_key()); - std::string bigbin(1000, 'x'); - CHECK_EQUAL(obj17.get(col_binary), BinaryData(bigbin)); - CHECK_EQUAL(obj17.get(col_date), Timestamp(1700, 17)); - CHECK_EQUAL(obj17.get(col_link), obj27.get_key()); - auto int_list_null = obj17.get_list(col_int_list); - CHECK(int_list_null.is_empty()); - - CHECK_EQUAL(obj18.get(col_string), ""); - CHECK_EQUAL(obj18.get(col_string_i), StringData()); - - auto int_list = obj23.get_list(col_int_list); - CHECK(!int_list.is_empty()); - CHECK_EQUAL(int_list.size(), 18); - CHECK_EQUAL(int_list.get(0), 24); - CHECK_EQUAL(int_list.get(17), 41); - - auto int_null_list = obj23.get_list>(col_int_null_list); - CHECK(!int_null_list.is_empty()); - CHECK_EQUAL(int_null_list.size(), 10); - CHECK_EQUAL(int_null_list.get(1), 5); - CHECK_NOT(int_null_list.get(5)); - - CHECK_EQUAL(obj27.get(col_bool), true); - std::string bin("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwx"); - CHECK_EQUAL(obj27.get(col_binary), BinaryData(bin)); - CHECK_EQUAL(obj27.get_backlink_count(*t, col_link), 1); - CHECK_EQUAL(obj27.get_backlink(*t, col_link, 0), obj17.get_key()); - auto ll = obj27.get_linklist(col_linklist); - CHECK_EQUAL(ll.size(), 7); - const Obj linked_obj = ll.get_object(0); - CHECK_EQUAL(linked_obj.get_key(), ObjKey(12)); - size_t expected_back_link_count = (REALM_MAX_BPNODE_SIZE == 4) ? 1 : 4; - CHECK_EQUAL(linked_obj.get_backlink_count(*t, col_linklist), expected_back_link_count); - - CHECK_EQUAL(obj.get(col_string_i), - "This is a rather long string, that should not be very much shorter"); - CHECK_EQUAL(obj.get(col_binary), BinaryData("", 0)); - - const Obj obj14 = o->get_object(14); - - auto str_list = obj14.get_list(col_str_list); - CHECK_EQUAL(str_list.size(), 3); - CHECK_EQUAL(str_list.get(0), StringData("Medium_Sized_String_0")); - CHECK_EQUAL(str_list.get(1), StringData("Medium_Sized_String_1")); - CHECK(str_list.is_null(2)); - - --iter; - } - if (REALM_MAX_BPNODE_SIZE == 1000) { - auto sg = DB::create(*hist, temp_copy); - if (generate_json) { - std::ofstream expect(test_util::get_test_path_prefix() + "expect_test_upgrade_database_9_to_10.json"); - sg->start_read()->to_json(expect, 0); - } - nlohmann::json expected; - nlohmann::json actual; - std::ifstream expect(test_util::get_test_resource_path() + "expect_test_upgrade_database_9_to_10.json"); - expect >> expected; - - std::stringstream out; - sg->start_read()->to_json(out, 0); - out >> actual; - CHECK(actual == expected); - } - -#else - // NOTE: This code must be executed from an old file-format-version 9 - // core in order to create a file-format-version 9 test file! - -#ifndef REALM_CLUSTER_IF - Group g; - TableRef t = g.add_table("table"); - TableRef o = g.add_table("other"); - g.add_table("empty"); - size_t col_int = t->add_column(type_Int, "int"); - size_t col_int_null = t->add_column(type_Int, "int", true); // Duplicate name - size_t col_bool = t->add_column(type_Bool, ""); // Missing name - size_t col_bool_null = t->add_column(type_Bool, "bool_null", true); - size_t col_float = t->add_column(type_Float, "float"); - size_t col_double = t->add_column(type_Double, "double"); - size_t col_string = t->add_column(type_String, "string"); - size_t col_string_i = t->add_column(type_String, "string_i", true); - size_t col_binary = t->add_column(type_Binary, "binary"); - size_t col_date = t->add_column(type_Timestamp, "date"); - size_t col_link = t->add_column_link(type_Link, "link", *t); - size_t col_linklist = t->add_column_link(type_LinkList, "linklist", *o); - - DescriptorRef subdesc; - size_t col_int_list = t->add_column(type_Table, "integers", false, &subdesc); - subdesc->add_column(type_Int, "list"); - - size_t col_int_null_list = t->add_column(type_Table, "intnulls", false, &subdesc); - subdesc->add_column(type_Int, "list", nullptr, true); - - size_t col_val = o->add_column(type_Int, "val"); - size_t col_string_list = o->add_column(type_Table, "strings", false, &subdesc); - subdesc->add_column(type_String, "list", nullptr, true); - - t->add_search_index(col_string_i); - - t->add_empty_row(nb_rows); - o->add_empty_row(25); - for (size_t i = 0; i < nb_rows; i++) { - if (i % 2) { - t->set_int(col_int, i, i); - t->set_int(col_int_null, i, i); - t->set_bool(col_bool, i, (i % 3) == 0); - t->set_bool(col_bool_null, i, (i % 3) == 0); - t->set_float(col_float, i, i * 1.5f); - t->set_double(col_double, i, i * 2.5); - - // String - std::string str = "foo "; - str += util::to_string(i); - t->set_string(col_string, i, str); - str = "This is a somewhat longer string"; - str += util::to_string(i); - t->set_string(col_string_i, i, str); - - // Binary - if (i % 9 == 0) { - const char bin[] = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; - t->set_binary(col_binary, i, BinaryData(bin, strlen(bin) - i / 10)); - } - - // Timestamp - t->set_timestamp(col_date, i, Timestamp(100 * i, i)); - - // Link - if ((i % 17) == 0) { - t->set_link(col_link, i, (i + 10) % nb_rows); - } - - // LinkList - if ((i % 27) == 0) { - auto lv = t->get_linklist(col_linklist, i); - for (size_t j = 0; j < i % 10; j++) { - lv->add(j + i % 15); - } - } - // ListOfPrimitives - if ((i % 23) == 0) { - auto st = t->get_subtable(col_int_list, i); - int nb_elements = 16 + i % 20; - st->add_empty_row(nb_elements); - for (int j = 0; j < nb_elements; j++) { - st->set_int(0, j, j + i % 30); - } - st->remove(0); - } - // ListOfOptionals - if (i == 23) { - auto st = t->get_subtable(col_int_null_list, i); - st->add_empty_row(10); - for (int j = 0; j < 10; j++) { - if (j != 5) - st->set_int(0, j, 5 * j); - } - } - } - } - - for (size_t i = 0; i < 25; i++) { - o->set_int(col_val, i, i); - auto st = o->get_subtable(col_string_list, i); - if (i % 5 == 0) { - for (size_t j = 0; j < i / 5; j++) { - st->add_empty_row(); - std::string str = "String_" + std::to_string(j); - st->set_string(0, j, str); - } - } - if (i % 7 == 0) { - for (size_t j = 0; j < i / 7; j++) { - st->add_empty_row(); - std::string str = "Medium_Sized_String_" + std::to_string(j); - st->set_string(0, j, str); - } - st->add_empty_row(); - } - } - - t->set_string(col_string, 17, "This is a medium long string"); - std::string bigbin(1000, 'x'); - t->set_binary(col_binary, 17, BinaryData(bigbin)); - t->insert_empty_row(insert_pos); - t->set_string(col_string_i, insert_pos, "This is a rather long string, that should not be very much shorter"); - - g.write(path); -#endif -#endif // TEST_READ_UPGRADE_MODE -} - TEST_IF(Upgrade_Database_10_11, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE == 1000) { std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + @@ -1094,30 +390,6 @@ TEST_IF(Upgrade_Database_20, REALM_MAX_BPNODE_SIZE == 1000) } } -TEST(Upgrade_progress) -{ - SHARED_GROUP_TEST_PATH(temp_copy); - auto hist = make_in_realm_history(); - - for (int i = 1; i <= 7; i++) { - auto fn = test_util::get_test_resource_path() + "test_upgrade_progress_" + util::to_string(i) + ".realm"; - File::copy(fn, temp_copy); - DB::create(*hist, temp_copy)->start_read()->verify(); - } -} - -TEST(Upgrade_FixColumnKeys) -{ - SHARED_GROUP_TEST_PATH(temp_copy); - // The "object" table in this file contains an m_keys array where the keys for the - // backlink columns are wrong. - auto fn = test_util::get_test_resource_path() + "test_upgrade_colkey_error.realm"; - File::copy(fn, temp_copy); - - auto hist = make_in_realm_history(); - DB::create(*hist, temp_copy)->start_read()->verify(); -} - NONCONCURRENT_TEST(Upgrade_BackupAtoBtoAtoC) { SHARED_GROUP_TEST_PATH(path); diff --git a/test/test_upgrade_database_1000_2.realm b/test/test_upgrade_database_1000_2.realm deleted file mode 100644 index fcb496973aa..00000000000 Binary files a/test/test_upgrade_database_1000_2.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_3.realm b/test/test_upgrade_database_1000_3.realm deleted file mode 100644 index 9dc89cfb0e0..00000000000 Binary files a/test/test_upgrade_database_1000_3.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_4.realm b/test/test_upgrade_database_1000_4.realm deleted file mode 100644 index d67e263eb6c..00000000000 Binary files a/test/test_upgrade_database_1000_4.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_4_to_5_datetime1.realm b/test/test_upgrade_database_1000_4_to_5_datetime1.realm deleted file mode 100644 index 8c63e19b1c8..00000000000 Binary files a/test/test_upgrade_database_1000_4_to_5_datetime1.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_5_to_6_stringindex.realm b/test/test_upgrade_database_1000_5_to_6_stringindex.realm deleted file mode 100644 index 33ef57ce407..00000000000 Binary files a/test/test_upgrade_database_1000_5_to_6_stringindex.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_6_to_7.realm b/test/test_upgrade_database_1000_6_to_7.realm deleted file mode 100644 index 3e67caa58bf..00000000000 Binary files a/test/test_upgrade_database_1000_6_to_7.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_7_to_8.realm b/test/test_upgrade_database_1000_7_to_8.realm deleted file mode 100644 index 4f06f91a4f1..00000000000 Binary files a/test/test_upgrade_database_1000_7_to_8.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_8_to_9.realm b/test/test_upgrade_database_1000_8_to_9.realm deleted file mode 100644 index dad3551a89e..00000000000 Binary files a/test/test_upgrade_database_1000_8_to_9.realm and /dev/null differ diff --git a/test/test_upgrade_database_1000_9_to_10.realm b/test/test_upgrade_database_1000_9_to_10.realm deleted file mode 100644 index a6abbd487e5..00000000000 Binary files a/test/test_upgrade_database_1000_9_to_10.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_1.realm b/test/test_upgrade_database_4_1.realm deleted file mode 100644 index 50ca6b69cd6..00000000000 Binary files a/test/test_upgrade_database_4_1.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_2.realm b/test/test_upgrade_database_4_2.realm deleted file mode 100644 index 1f1595f551c..00000000000 Binary files a/test/test_upgrade_database_4_2.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_3.realm b/test/test_upgrade_database_4_3.realm deleted file mode 100644 index 9dc89cfb0e0..00000000000 Binary files a/test/test_upgrade_database_4_3.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_4.realm b/test/test_upgrade_database_4_4.realm deleted file mode 100644 index 9bf16806a87..00000000000 Binary files a/test/test_upgrade_database_4_4.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_4_to_5_datetime1.realm b/test/test_upgrade_database_4_4_to_5_datetime1.realm deleted file mode 100644 index 8c63e19b1c8..00000000000 Binary files a/test/test_upgrade_database_4_4_to_5_datetime1.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_6_to_7.realm b/test/test_upgrade_database_4_6_to_7.realm deleted file mode 100644 index b781a3f3ea5..00000000000 Binary files a/test/test_upgrade_database_4_6_to_7.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_7_to_8.realm b/test/test_upgrade_database_4_7_to_8.realm deleted file mode 100644 index f1066362c75..00000000000 Binary files a/test/test_upgrade_database_4_7_to_8.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_8_to_9.realm b/test/test_upgrade_database_4_8_to_9.realm deleted file mode 100644 index 83556ae63b4..00000000000 Binary files a/test/test_upgrade_database_4_8_to_9.realm and /dev/null differ diff --git a/test/test_upgrade_database_4_9_to_10.realm b/test/test_upgrade_database_4_9_to_10.realm deleted file mode 100644 index bfd5ff1babf..00000000000 Binary files a/test/test_upgrade_database_4_9_to_10.realm and /dev/null differ diff --git a/test/test_upgrade_database_6.realm b/test/test_upgrade_database_6.realm deleted file mode 100644 index 352f0894db4..00000000000 Binary files a/test/test_upgrade_database_6.realm and /dev/null differ diff --git a/test/test_upgrade_database_9_to_10_pk_table.realm b/test/test_upgrade_database_9_to_10_pk_table.realm deleted file mode 100644 index 3750bf1e1b0..00000000000 Binary files a/test/test_upgrade_database_9_to_10_pk_table.realm and /dev/null differ diff --git a/test/test_upgrade_progress_1.realm b/test/test_upgrade_progress_1.realm deleted file mode 100644 index d6d98433f0b..00000000000 Binary files a/test/test_upgrade_progress_1.realm and /dev/null differ diff --git a/test/test_upgrade_progress_2.realm b/test/test_upgrade_progress_2.realm deleted file mode 100644 index f5c9a2f4662..00000000000 Binary files a/test/test_upgrade_progress_2.realm and /dev/null differ diff --git a/test/test_upgrade_progress_3.realm b/test/test_upgrade_progress_3.realm deleted file mode 100644 index 6079a7f1d24..00000000000 Binary files a/test/test_upgrade_progress_3.realm and /dev/null differ diff --git a/test/test_upgrade_progress_4.realm b/test/test_upgrade_progress_4.realm deleted file mode 100644 index 30e4e011909..00000000000 Binary files a/test/test_upgrade_progress_4.realm and /dev/null differ diff --git a/test/test_upgrade_progress_5.realm b/test/test_upgrade_progress_5.realm deleted file mode 100644 index 6be8ef8b434..00000000000 Binary files a/test/test_upgrade_progress_5.realm and /dev/null differ diff --git a/test/test_upgrade_progress_6.realm b/test/test_upgrade_progress_6.realm deleted file mode 100644 index 00c3e29453e..00000000000 Binary files a/test/test_upgrade_progress_6.realm and /dev/null differ diff --git a/test/test_upgrade_progress_7.realm b/test/test_upgrade_progress_7.realm deleted file mode 100644 index 13b77831ca4..00000000000 Binary files a/test/test_upgrade_progress_7.realm and /dev/null differ