diff options
author | Franciszek Malinka <franciszek.malinka@gmail.com> | 2022-11-07 20:31:46 +0100 |
---|---|---|
committer | Franciszek Malinka <franciszek.malinka@gmail.com> | 2022-11-10 18:09:31 +0100 |
commit | 0f0172f850c72d0561ad00f6f95536d28efe0d0d (patch) | |
tree | d5b8b191551094281f20f15e26bf3210bfb5500e | |
parent | f70e0f7271a913ec220e31a86b827adee233968f (diff) |
Added database.cc with LoadSchema factory constructor.
Also:
- Added database_test (which gave some bugs),
- Added some logging to other files,
- Expanded some error messages with relevant data,
- Moved test data to a separate catalog.
-rw-r--r-- | komfydb/BUILD | 40 | ||||
-rw-r--r-- | komfydb/common/BUILD | 2 | ||||
-rw-r--r-- | komfydb/common/database.cc | 0 | ||||
-rw-r--r-- | komfydb/common/database.h | 0 | ||||
-rw-r--r-- | komfydb/common/tuple.cc | 4 | ||||
-rw-r--r-- | komfydb/database.cc | 141 | ||||
-rw-r--r-- | komfydb/database.h | 33 | ||||
-rw-r--r-- | komfydb/database_test.cc | 17 | ||||
-rw-r--r-- | komfydb/komfydb.cc | 29 | ||||
-rw-r--r-- | komfydb/storage/BUILD | 3 | ||||
-rw-r--r-- | komfydb/storage/buffer_pool.h | 2 | ||||
-rw-r--r-- | komfydb/storage/catalog.cc | 12 | ||||
-rw-r--r-- | komfydb/storage/heap_file.cc | 10 | ||||
-rw-r--r-- | komfydb/storage/heap_file.h | 2 | ||||
-rw-r--r-- | komfydb/storage/heap_file_test.cc | 6 | ||||
-rw-r--r-- | komfydb/storage/heap_page.cc | 4 | ||||
-rw-r--r-- | komfydb/storage/heap_page.h | 2 | ||||
-rw-r--r-- | komfydb/storage/testdata/heap_file_test.dat (renamed from komfydb/storage/heap_file_test.dat) | bin | 274432 -> 274432 bytes | |||
-rw-r--r-- | komfydb/testdata/database_catalog_test.txt | 2 | ||||
-rw-r--r-- | komfydb/testdata/first_table.dat | bin | 0 -> 274432 bytes | |||
-rw-r--r-- | komfydb/testdata/second_table.dat | bin | 0 -> 274432 bytes |
21 files changed, 289 insertions, 20 deletions
diff --git a/komfydb/BUILD b/komfydb/BUILD index 4aa2eca..112f947 100644 --- a/komfydb/BUILD +++ b/komfydb/BUILD @@ -14,10 +14,50 @@ cc_library( visibility = ["//visibility:public"], ) +cc_library( + name = "database", + srcs = [ + "database.cc", + ], + hdrs = [ + "database.h", + ], + deps = [ + "//komfydb/storage", + "//komfydb/common", + "@com_github_google_glog//:glog", + ], +) + +filegroup( + name = "database_test_data", + srcs = glob(["testdata/*"]), +) + +cc_test( + name = "database_test", + srcs = ["database_test.cc"], + data = [":database_test_data"], + deps = [ + "//komfydb:database", + "//komfydb/storage:storage", + "//komfydb/common:common", + "//komfydb/transaction:transaction", + "//komfydb/execution:execution", + "@com_google_googletest//:gtest_main", + ], +) + cc_binary( name = "komfydb", srcs = ["komfydb.cc"], + data = [ + "testdata/database_catalog_test.txt", + "testdata/first_table.dat", + "testdata/second_table.dat", + ], deps = [ + "//komfydb:database", "//komfydb/storage:storage", "//komfydb/common:common", "//komfydb/transaction:transaction", diff --git a/komfydb/common/BUILD b/komfydb/common/BUILD index 39cd04b..667a50c 100644 --- a/komfydb/common/BUILD +++ b/komfydb/common/BUILD @@ -61,12 +61,10 @@ cc_test( cc_library( name = "common", hdrs = [ - "database.h", "debug.h", "permissions.h", ], srcs = [ - "database.cc", "debug.cc", ], deps = [ diff --git a/komfydb/common/database.cc b/komfydb/common/database.cc deleted file mode 100644 index e69de29..0000000 --- a/komfydb/common/database.cc +++ /dev/null diff --git a/komfydb/common/database.h b/komfydb/common/database.h deleted file mode 100644 index e69de29..0000000 --- a/komfydb/common/database.h +++ /dev/null diff --git a/komfydb/common/tuple.cc b/komfydb/common/tuple.cc index 4658260..e821aee 100644 --- a/komfydb/common/tuple.cc +++ b/komfydb/common/tuple.cc @@ -42,10 +42,10 @@ absl::Status Tuple::SetField(int i, std::unique_ptr<Field> f) { Tuple::operator std::string() const { std::string res = ""; for (int i = 0; i < fields.size() - 1; i++) { - res += (std::string)*fields[i]; + res += static_cast<std::string>(*fields[i]); res += " "; } - res += (std::string)*fields.back(); + res += static_cast<std::string>(*fields.back()); return res; } diff --git a/komfydb/database.cc b/komfydb/database.cc new file mode 100644 index 0000000..4161a66 --- /dev/null +++ b/komfydb/database.cc @@ -0,0 +1,141 @@ +#include "komfydb/database.h" + +#include <filesystem> +#include <fstream> +#include <string> + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "glog/logging.h" + +#include "komfydb/common/tuple_desc.h" +#include "komfydb/common/type.h" +#include "komfydb/storage/db_file.h" +#include "komfydb/storage/heap_file.h" +#include "komfydb/utils/status_macros.h" + +namespace { + +using komfydb::common::TupleDesc; +using komfydb::common::Type; +using komfydb::storage::DbFile; +using komfydb::storage::HeapFile; + +inline void ltrim(std::string& s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { + return !std::isspace(ch); + })); +} + +// trim from end (in place) +inline void rtrim(std::string& s) { + s.erase(std::find_if(s.rbegin(), s.rend(), + [](unsigned char ch) { return !std::isspace(ch); }) + .base(), + s.end()); +} + +// trim from both ends (in place) +inline void trim(std::string& s) { + ltrim(s); + rtrim(s); +} + +inline void ToLower(std::string& s) { + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return std::tolower(c); }); +} + +std::vector<std::string> SplitLine(std::string line, char delim) { + std::vector<std::string> tokens; + std::stringstream line_stream(line); + std::string token; + + while (getline(line_stream, token, delim)) { + trim(token); + tokens.push_back(token); + } + return tokens; +} + +absl::StatusOr<Type> GetType(std::string& type_str) { + ToLower(type_str); + if (type_str == "int") { + return Type::INT; + } else if (type_str == "string") { + return Type::STRING; + } else { + return absl::InvalidArgumentError( + absl::StrCat("Cannot parse type: ", type_str)); + } +} + +}; // namespace + +namespace komfydb { + +Database::Database(std::shared_ptr<Catalog> catalog) + : catalog(std::move(catalog)), buffer_pool(catalog) {} + +absl::StatusOr<Database> Database::LoadSchema( + absl::string_view catalog_file_path) { + std::string directory = + std::filesystem::path(catalog_file_path).parent_path().string(); + std::fstream catalog_file; + catalog_file.open((std::string)catalog_file_path, std::ios::in); + if (!catalog_file.good()) { + return absl::InvalidArgumentError( + absl::StrCat("Cannot open catalog file: ", catalog_file_path)); + } + + std::string line; + std::shared_ptr<Catalog> catalog = std::make_shared<Catalog>(); + + while (std::getline(catalog_file, line)) { + std::string name = line.substr(0, line.find("(")); + line = line.substr(line.find("(") + 1); + line.pop_back(); // delete last ')' + trim(name); + std::vector<std::string> tokens = SplitLine(line, ','); + std::vector<std::string> names; + std::vector<Type> types; + std::string primary_key = ""; + for (auto token : tokens) { + LOG(INFO) << "Parsing token " << token; + trim(token); + std::vector<std::string> parts = SplitLine(token, ' '); + if (parts.size() < 2 || parts.size() > 4) { + return absl::InvalidArgumentError( + absl::StrCat("Cannot parse field description: ", token)); + } + ASSIGN_OR_RETURN(Type type, GetType(parts[1])); + types.push_back(type); + trim(parts[0]); + names.push_back(parts[0]); + if (parts.size() == 3) { + trim(parts[2]); + ToLower(parts[2]); + if (parts[2] == "pk") { + primary_key = parts[0]; + } else { + return absl::InvalidArgumentError( + absl::StrCat("Unknown field attribute: ", parts[2])); + } + } + } + + TupleDesc td(types, names); + ASSIGN_OR_RETURN(std::unique_ptr<HeapFile> hp, + HeapFile::Create(directory + "/" + name + ".dat", td, + Permissions::READ_ONLY)); + catalog->AddTable(std::move(hp), name, primary_key); + } + + return Database(std::move(catalog)); +} + +std::shared_ptr<Catalog> Database::GetCatalog() { + return catalog; +} + +}; // namespace komfydb diff --git a/komfydb/database.h b/komfydb/database.h new file mode 100644 index 0000000..94bc013 --- /dev/null +++ b/komfydb/database.h @@ -0,0 +1,33 @@ +#ifndef __DATABASE_H__ +#define __DATABASE_H__ + +#include "absl/status/statusor.h" + +#include "komfydb/storage/buffer_pool.h" +#include "komfydb/storage/catalog.h" + +namespace { + +using komfydb::storage::BufferPool; +using komfydb::storage::Catalog; + +}; // namespace + +namespace komfydb { + +class Database { + private: + std::shared_ptr<Catalog> catalog; + BufferPool buffer_pool; + + Database(std::shared_ptr<Catalog> catalog); + + public: + static absl::StatusOr<Database> LoadSchema(absl::string_view catalog_file); + + std::shared_ptr<Catalog> GetCatalog(); +}; + +}; // namespace komfydb + +#endif // __DATABASE_H__ diff --git a/komfydb/database_test.cc b/komfydb/database_test.cc new file mode 100644 index 0000000..9844fef --- /dev/null +++ b/komfydb/database_test.cc @@ -0,0 +1,17 @@ +#include "gtest/gtest.h" + +#include "absl/status/statusor.h" + +#include "komfydb/database.h" + +namespace { + +using komfydb::Database; + +TEST(Database, LoadSchema) { + absl::StatusOr<Database> db = + Database::LoadSchema("komfydb/testdata/database_catalog_test.txt"); + ASSERT_TRUE(db.ok()); +} + +}; // namespace diff --git a/komfydb/komfydb.cc b/komfydb/komfydb.cc index a8c844a..cb734a9 100644 --- a/komfydb/komfydb.cc +++ b/komfydb/komfydb.cc @@ -1,11 +1,15 @@ #include <iostream> #include <string> -#include "common/type.h" +#include "absl/status/statusor.h" #include "glog/logging.h" +#include "common/type.h" #include "komfydb/common/td_item.h" #include "komfydb/common/type.h" +#include "komfydb/database.h" +#include "komfydb/storage/heap_page.h" +#include "utils/status_macros.h" using namespace komfydb; @@ -15,8 +19,25 @@ int main(int argc, char* argv[]) { LOG(INFO) << "Welcome to KonfyDB!"; - common::Type t(common::Type::Value::INT); - common::TDItem tditem(t, "name"); + absl::StatusOr<Database> db = + Database::LoadSchema("komfydb/testdata/database_catalog_test.txt"); + if (!db.ok()) { + LOG(ERROR) << "LoadSchema error: " << db.status().message(); + } + + std::shared_ptr<Catalog> catalog = db->GetCatalog(); + std::vector<int> table_ids = catalog->GetTableIds(); + + LOG(INFO) << "Tables:"; + for (auto table_id : table_ids) { + LOG(INFO) << table_id << "->" << catalog->GetTableName(table_id).value(); + auto file = catalog->GetDatabaseFile(table_id).value(); + std::unique_ptr<storage::Page> page = + file->ReadPage(storage::PageId(table_id, 0)).value(); + storage::HeapPage* hp = static_cast<storage::HeapPage*>(page.get()); - LOG(INFO) << "Created tditem: " << (std::string)tditem; + for (auto& tuple : hp->GetTuples()) { + LOG(INFO) << static_cast<std::string>(tuple); + } + } } diff --git a/komfydb/storage/BUILD b/komfydb/storage/BUILD index 2ed64a1..35b42cd 100644 --- a/komfydb/storage/BUILD +++ b/komfydb/storage/BUILD @@ -28,6 +28,7 @@ cc_library( "//komfydb/utils:utility", "//komfydb/common:common", "//komfydb:config", + "@com_github_google_glog//:glog", "@com_google_absl//absl/status", "@com_google_absl//absl/container:flat_hash_map", ], @@ -37,7 +38,7 @@ cc_library( cc_test( name = "heap_file_test", srcs = ["heap_file_test.cc"], - data = ["heap_file_test.dat"], + data = ["testdata/heap_file_test.dat"], deps = [ ":storage", "@com_google_googletest//:gtest_main", diff --git a/komfydb/storage/buffer_pool.h b/komfydb/storage/buffer_pool.h index 203efb5..5703b05 100644 --- a/komfydb/storage/buffer_pool.h +++ b/komfydb/storage/buffer_pool.h @@ -31,7 +31,7 @@ class BufferPool { public: BufferPool(std::shared_ptr<Catalog> catalog, int pages_cnt = PAGES_CNT) - : pages_cnt(pages_cnt), catalog(catalog) {} + : pages_cnt(pages_cnt), catalog(std::move(catalog)) {} int GetPageSize() const { return pages_cnt; } diff --git a/komfydb/storage/catalog.cc b/komfydb/storage/catalog.cc index 0a5c2b1..ce2d92a 100644 --- a/komfydb/storage/catalog.cc +++ b/komfydb/storage/catalog.cc @@ -1,7 +1,9 @@ +#include "komfydb/storage/catalog.h" + #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" +#include "glog/logging.h" -#include "komfydb/storage/catalog.h" #include "komfydb/utils/status_macros.h" #include "komfydb/utils/utility.h" @@ -11,7 +13,8 @@ template <typename K, typename V> absl::StatusOr<V> StatusOrMapElement(const absl::flat_hash_map<K, V>& map, const K& key) { if (!map.contains(key)) { - return absl::InvalidArgumentError("No element for given key."); + return absl::InvalidArgumentError( + absl::StrCat("No element for given key=", key)); } return map.at(key); } @@ -22,6 +25,8 @@ namespace komfydb::storage { void Catalog::AddTable(std::unique_ptr<DbFile> file, std::string name, std::string primary_key) { + LOG(INFO) << "Adding table: tid=" << file->GetId() << " name=" << name + << " pk=" << primary_key; int id = file->GetId(); db_files[id] = std::move(file); @@ -49,7 +54,8 @@ absl::StatusOr<std::string> Catalog::GetTableName(int table_id) const { absl::StatusOr<DbFile*> Catalog::GetDatabaseFile(int table_id) const { auto it = db_files.find(table_id); if (it == db_files.end()) { - return absl::InvalidArgumentError("No table with given name"); + return absl::InvalidArgumentError( + absl::StrCat("No table with id=", table_id)); } return it->second.get(); } diff --git a/komfydb/storage/heap_file.cc b/komfydb/storage/heap_file.cc index dec3918..4f43317 100644 --- a/komfydb/storage/heap_file.cc +++ b/komfydb/storage/heap_file.cc @@ -35,7 +35,8 @@ absl::StatusOr<std::unique_ptr<HeapFile>> HeapFile::Create( std::fstream file; file.open(std::string(file_path), mode); if (!file.good()) { - return absl::InvalidArgumentError("Could not open specified db file."); + return absl::InvalidArgumentError( + absl::StrCat("Could not open db file: ", file_path)); } file.seekg(0, file.end); @@ -63,14 +64,17 @@ TupleDesc* HeapFile::GetTupleDesc() { absl::StatusOr<std::unique_ptr<Page>> HeapFile::ReadPage(PageId id) { if (id.GetTableId() != table_id) { - return absl::InvalidArgumentError("Table ID does not match."); + return absl::InvalidArgumentError(absl::StrCat( + "Table ID does not match: ", table_id, "!=", id.GetTableId())); } file.seekg(0, file.end); uint32_t file_length = file.tellg(); uint64_t page_pos = (uint64_t)CONFIG_PAGE_SIZE * (uint64_t)id.GetPageNumber(); if (page_pos >= file_length) { - return absl::InvalidArgumentError("Page number out of range."); + return absl::InvalidArgumentError( + absl::StrCat("Page number out of range: ", id.GetPageNumber(), " (", + file_length / CONFIG_PAGE_SIZE, ")")); } std::vector<uint8_t> data(CONFIG_PAGE_SIZE); diff --git a/komfydb/storage/heap_file.h b/komfydb/storage/heap_file.h index 9ccd494..dcfd786 100644 --- a/komfydb/storage/heap_file.h +++ b/komfydb/storage/heap_file.h @@ -27,7 +27,7 @@ using komfydb::common::TupleDesc; namespace komfydb::storage { -class HeapFile : DbFile { +class HeapFile : public DbFile { private: std::fstream file; TupleDesc td; diff --git a/komfydb/storage/heap_file_test.cc b/komfydb/storage/heap_file_test.cc index 0ad4dd4..b5c1e0e 100644 --- a/komfydb/storage/heap_file_test.cc +++ b/komfydb/storage/heap_file_test.cc @@ -15,7 +15,7 @@ using komfydb::common::Type; class HeapFileTest : public ::testing::Test { protected: - const char* kTestDataFilePath = "komfydb/storage/heap_file_test.dat"; + const char* kTestDataFilePath = "komfydb/storage/testdata/heap_file_test.dat"; const int tuples = 1000; int tuple_sz; int pages_cnt; @@ -49,11 +49,11 @@ TEST_F(HeapFileTest, ReadPageErrors) { page = hp->ReadPage(PageId(table_id, pages_cnt + 1)); ASSERT_FALSE(page.ok()); - EXPECT_EQ(page.status().message(), "Page number out of range."); + EXPECT_EQ(page.status().message(), "Page number out of range: 67 (67)"); page = hp->ReadPage(PageId(table_id + 1, 0)); ASSERT_FALSE(page.ok()); - EXPECT_EQ(page.status().message(), "Table ID does not match."); + EXPECT_EQ(page.status().message(), "Table ID does not match: 1!=2"); } TEST_F(HeapFileTest, ReadPage) { diff --git a/komfydb/storage/heap_page.cc b/komfydb/storage/heap_page.cc index fb19c9b..1d5decc 100644 --- a/komfydb/storage/heap_page.cc +++ b/komfydb/storage/heap_page.cc @@ -129,4 +129,8 @@ absl::Status HeapPage::SetBeforeImage() { return absl::OkStatus(); } +std::vector<Tuple>& HeapPage::GetTuples() { + return tuples; +} + }; // namespace komfydb::storage diff --git a/komfydb/storage/heap_page.h b/komfydb/storage/heap_page.h index d87f79e..d3d5307 100644 --- a/komfydb/storage/heap_page.h +++ b/komfydb/storage/heap_page.h @@ -62,6 +62,8 @@ class HeapPage : public Page { absl::StatusOr<std::unique_ptr<Page>> GetBeforeImage() override; absl::StatusOr<std::vector<uint8_t>> GetPageData() override; + + std::vector<Tuple>& GetTuples(); }; }; // namespace komfydb::storage diff --git a/komfydb/storage/heap_file_test.dat b/komfydb/storage/testdata/heap_file_test.dat Binary files differindex 971e8bd..971e8bd 100644 --- a/komfydb/storage/heap_file_test.dat +++ b/komfydb/storage/testdata/heap_file_test.dat diff --git a/komfydb/testdata/database_catalog_test.txt b/komfydb/testdata/database_catalog_test.txt new file mode 100644 index 0000000..07d72ed --- /dev/null +++ b/komfydb/testdata/database_catalog_test.txt @@ -0,0 +1,2 @@ +first_table (name1 int, name2 string, name3 int pk, name4 string) +second_table (name1 int, name2 string, name3 int pk, name4 string) diff --git a/komfydb/testdata/first_table.dat b/komfydb/testdata/first_table.dat Binary files differnew file mode 100644 index 0000000..971e8bd --- /dev/null +++ b/komfydb/testdata/first_table.dat diff --git a/komfydb/testdata/second_table.dat b/komfydb/testdata/second_table.dat Binary files differnew file mode 100644 index 0000000..971e8bd --- /dev/null +++ b/komfydb/testdata/second_table.dat |