Skip to content

Commit

Permalink
Try adding test db with git-lfs
Browse files Browse the repository at this point in the history
  • Loading branch information
jeffreypicard committed Dec 7, 2021
1 parent 2ee8c62 commit a2c35c8
Show file tree
Hide file tree
Showing 16 changed files with 951 additions and 10 deletions.
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
resources/ filter=lfs diff=lfs merge=lfs -text
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.idea/
10 changes: 8 additions & 2 deletions Dockerfile.action
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
FROM jeffreypicard/hub-github-env

COPY scripts/build_and_test.sh /build_and_test.sh
# COPY scripts/build_and_test.sh /build_and_test.sh
COPY . /hub
WORKDIR /hub

ENTRYPOINT ["/build_and_test.sh"]
ENV CGO_LDFLAGS "-L/usr/local/lib -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd"
ENV CGO_CFLAGS "-I/usr/local/include/rocksdb"
ENV LD_LIBRARY_PATH /usr/local/lib

ENTRYPOINT ["scripts/build_and_test.sh"]
49 changes: 48 additions & 1 deletion db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,57 @@ import (
"log"
)

func OpenDB(name string) {
func OpenDB(name string) int {
// Read db
opts := grocksdb.NewDefaultOptions()
db, err := grocksdb.OpenDb(opts, name)
ro := grocksdb.NewDefaultReadOptions()
ro.SetFillCache(false)
if err != nil {
log.Println(err)
}

log.Println(db.Name())

it := db.NewIterator(ro)
defer it.Close()

var i = 0
it.Seek([]byte("foo"))
for it = it; it.Valid() && i < 10; it.Next() {
key := it.Key()
value := it.Value()

fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data())

key.Free()
value.Free()
i++
}
if err := it.Err(); err != nil {
log.Println(err)
}

return i
}

func OpenAndWriteDB(in string, out string) {
// Read db
opts := grocksdb.NewDefaultOptions()
db, err := grocksdb.OpenDb(opts, in)
ro := grocksdb.NewDefaultReadOptions()
ro.SetFillCache(false)
if err != nil {
log.Println(err)
}
// Write db
opts.SetCreateIfMissing(true)
db2, err := grocksdb.OpenDb(opts, out)
wo := grocksdb.NewDefaultWriteOptions()
defer db2.Close()

log.Println(db.Name())
log.Println(db2.Name())

it := db.NewIterator(ro)
defer it.Close()
Expand All @@ -25,6 +67,11 @@ func OpenDB(name string) {
key := it.Key()
value := it.Value()
fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data())

if err := db2.Put(wo, key.Data(), value.Data()); err != nil {
log.Println(err)
}

key.Free()
value.Free()
i++
Expand Down
10 changes: 5 additions & 5 deletions db/db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,21 @@ func TestOpenDB(t *testing.T) {

tests := []struct {
name string
want string
want int
}{
{
name: "Open a rocksdb database",
want: "",
want: 10,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
OpenDB("/mnt/d/data/wallet/lbry-rocksdb/")
got := ""
vals := OpenDB("../resources/tmp.db")
got := vals

if got != tt.want {
t.Errorf("got: %s, want: %s\n", got, tt.want)
t.Errorf("got: %d, want: %d\n", got, tt.want)
}
})
}
Expand Down
Binary file added resources/tmp.db/000009.sst
Binary file not shown.
Empty file added resources/tmp.db/000011.log
Empty file.
1 change: 1 addition & 0 deletions resources/tmp.db/CURRENT
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
MANIFEST-000010
1 change: 1 addition & 0 deletions resources/tmp.db/IDENTITY
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b6f61266-9f2d-4712-a981-584bd5d6e0b3
Empty file added resources/tmp.db/LOCK
Empty file.
278 changes: 278 additions & 0 deletions resources/tmp.db/LOG

Large diffs are not rendered by default.

234 changes: 234 additions & 0 deletions resources/tmp.db/LOG.old.1638876786538284

Large diffs are not rendered by default.

Binary file added resources/tmp.db/MANIFEST-000010
Binary file not shown.
187 changes: 187 additions & 0 deletions resources/tmp.db/OPTIONS-000007
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#

[Version]
rocksdb_version=6.26.1
options_file_version=1.1

[DBOptions]
max_open_files=-1
stats_history_buffer_size=1048576
stats_persist_period_sec=600
max_background_flushes=-1
stats_dump_period_sec=600
compaction_readahead_size=0
bytes_per_sync=0
delete_obsolete_files_period_micros=21600000000
max_total_wal_size=0
delayed_write_rate=16777216
wal_bytes_per_sync=0
writable_file_max_buffer_size=1048576
avoid_flush_during_shutdown=false
max_subcompactions=1
strict_bytes_per_sync=false
max_background_compactions=-1
base_background_compactions=-1
max_background_jobs=2
file_checksum_gen_factory=nullptr
db_host_id=__hostname__
bgerror_resume_retry_interval=1000000
best_efforts_recovery=false
avoid_unnecessary_blocking_io=false
two_write_queues=false
atomic_flush=false
preserve_deletes=false
allow_ingest_behind=false
lowest_used_cache_tier=kNonVolatileBlockTier
avoid_flush_during_recovery=false
info_log_level=INFO_LEVEL
access_hint_on_compaction_start=NORMAL
max_bgerror_resume_count=2147483647
write_thread_slow_yield_usec=3
allow_concurrent_memtable_write=true
WAL_ttl_seconds=0
manual_wal_flush=false
dump_malloc_stats=false
wal_recovery_mode=kPointInTimeRecovery
log_file_time_to_roll=0
enable_write_thread_adaptive_yield=true
recycle_log_file_num=0
table_cache_numshardbits=6
max_file_opening_threads=16
allow_data_in_errors=false
use_fsync=false
unordered_write=false
fail_if_options_file_error=false
random_access_max_buffer_size=1048576
new_table_reader_for_compaction_inputs=false
skip_checking_sst_file_sizes_on_db_open=false
skip_stats_update_on_db_open=false
persist_stats_to_disk=false
track_and_verify_wals_in_manifest=false
enable_pipelined_write=false
flush_verify_memtable_count=true
log_readahead_size=0
is_fd_close_on_exec=true
WAL_size_limit_MB=0
experimental_mempurge_threshold=0.000000
write_dbid_to_manifest=false
use_adaptive_mutex=false
error_if_exists=false
write_thread_max_yield_usec=100
enable_thread_tracking=false
db_write_buffer_size=0
create_missing_column_families=false
paranoid_checks=true
create_if_missing=true
wal_filter=nullptr
max_manifest_file_size=1073741824
allow_2pc=false
use_direct_io_for_flush_and_compaction=false
manifest_preallocation_size=4194304
use_direct_reads=false
allow_fallocate=true
max_write_batch_group_size_bytes=1048576
keep_log_file_num=1000
allow_mmap_reads=false
max_log_file_size=0
allow_mmap_writes=false
advise_random_on_open=true


[CFOptions "default"]
bottommost_compression=kDisableCompressionOption
bottommost_compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;}
sample_for_compression=0
blob_garbage_collection_force_threshold=1.000000
blob_file_size=268435456
compaction_options_universal={allow_trivial_move=false;incremental=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;min_merge_width=2;size_ratio=1;}
compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;}
prefix_extractor=nullptr
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
max_bytes_for_level_base=268435456
memtable_whole_key_filtering=false
memtable_prefix_bloom_size_ratio=0.000000
enable_blob_files=false
target_file_size_base=67108864
memtable_huge_page_size=0
max_successive_merges=0
inplace_update_num_locks=10000
max_sequential_skip_in_iterations=8
arena_block_size=1048576
target_file_size_multiplier=1
max_write_buffer_number=2
write_buffer_size=67108864
blob_compression_type=kNoCompression
compression=kSnappyCompression
level0_stop_writes_trigger=36
level0_slowdown_writes_trigger=20
level0_file_num_compaction_trigger=4
ttl=2592000
max_compaction_bytes=1677721600
blob_garbage_collection_age_cutoff=0.250000
compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;}
enable_blob_garbage_collection=false
soft_pending_compaction_bytes_limit=68719476736
paranoid_file_checks=false
periodic_compaction_seconds=0
check_flush_compaction_key_order=true
min_blob_size=0
hard_pending_compaction_bytes_limit=274877906944
disable_auto_compactions=false
max_bytes_for_level_multiplier=10.000000
report_bg_io_stats=false
compaction_pri=kMinOverlappingRatio
compaction_filter_factory=nullptr
comparator=leveldb.BytewiseComparator
sst_partitioner_factory=nullptr
bloom_locality=0
compaction_style=kCompactionStyleLevel
min_write_buffer_number_to_merge=1
max_write_buffer_size_to_maintain=0
max_write_buffer_number_to_maintain=0
merge_operator=nullptr
memtable_factory=SkipListFactory
memtable_insert_with_hint_prefix_extractor=nullptr
num_levels=7
force_consistency_checks=true
optimize_filters_for_hits=false
compaction_filter=nullptr
level_compaction_dynamic_level_bytes=false
inplace_update_support=false
table_factory=BlockBasedTable

[TableOptions/BlockBasedTable "default"]
pin_top_level_index_and_filter=true
block_align=false
read_amp_bytes_per_bit=0
verify_compression=false
enable_index_compression=true
whole_key_filtering=true
max_auto_readahead_size=262144
optimize_filters_for_memory=false
index_block_restart_interval=1
prepopulate_block_cache=kDisable
block_restart_interval=16
block_size=4096
format_version=5
partition_filters=false
block_size_deviation=10
no_block_cache=false
checksum=kCRC32c
data_block_hash_table_util_ratio=0.750000
index_shortening=kShortenSeparators
data_block_index_type=kDataBlockBinarySearch
hash_index_allow_collision=true
filter_policy=nullptr
metadata_block_size=4096
index_type=kBinarySearch
metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;}
pin_l0_filter_and_index_blocks_in_cache=false
cache_index_and_filter_blocks_with_high_priority=true
cache_index_and_filter_blocks=false
flush_block_policy_factory=FlushBlockBySizePolicyFactory

Loading

0 comments on commit a2c35c8

Please sign in to comment.