提交 39bb7d78 编写于 作者: Q qq_36105691

fix

上级 5eadeb5d
......@@ -599,32 +599,53 @@ players
players
```
这样的输入,所以可以使用容器来缓存结果,减少磁盘IO。
这样的输入,所以可以使用容器来缓存结果,减少读取数据库的磁盘IO。
```cpp
class MatchManager {
private:
static std::unordered_map<std::string, std::vector<MatchResult>> matchResultsByNames;
static std::unordered_map<std::string, std::vector<MatchResultDetail>> matchDetailsByNames;
private:
static std::unordered_map<std::string, ByteStream> matchResultsByNames;
static std::unordered_map<std::string, ByteStream> matchDetailsByNames;
public:
MatchManager() = delete;
public:
MatchManager() = delete;
~MatchManager() = delete;
~MatchManager() = delete;
MatchManager(const MatchManager &) = delete;
MatchManager(const MatchManager &) = delete;
MatchManager &operator=(const MatchManager &) = delete;
MatchManager &operator=(const MatchManager &) = delete;
static const std::vector<MatchResultDetail> &getMatchDetailsByName(const std::string &name);
static ByteStream getMatchDetailsByName(const std::string &name);
static const std::vector<MatchResult> &getMatchResultsByName(const std::string &name);
static ByteStream getMatchResultsByName(const std::string &name);
};
ByteStream MatchManager::getMatchDetailsByName(const std::string &name) {
if (matchDetailsByNames.find(name) == matchDetailsByNames.end()) {
auto temp = MatchDAO::getMatchResultsDetail(name);
std::string tempString = {};
for (const auto &result: temp) {
tempString += "Full Name:" + result.getfullName() + "\n";
tempString += "Rank:" + result.getrank() + "\n";
tempString += "Preliminary Score:" + result.getpreliminaryScore() + "\n";
tempString += "Semifinal Score:" + result.getsemifinalScore() + "\n";
tempString += "Final Score:" + result.getfinalScore() + "\n";
tempString += "-----\n";
}
char *data = new char[tempString.size()];
memcpy(data, tempString.data(), tempString.size());
matchDetailsByNames[name] = {data, tempString.size()};
}
return matchDetailsByNames[name];
}
```
这里是使用了`std::unordered_map`来作为缓存容器。
这里是使用了`std::unordered_map`来作为缓存容器,缓存可以直接写入文件的字节流,节省文件多批次写入带来的性能浪费
我使用了**8000**条命令进行测试,测试结果如下:
我使用了**8000**条命令进行测试(使用ofstream写入),测试结果如下:
**缓存前**
......@@ -638,30 +659,7 @@ public:
如果需要进一步提升性能,也许只能从磁盘IO入手了。
### 6.3 IO
`fstream` 是一个比较慢的输出流,所以可以使用`fwrite`来进行优化。
```cpp
void ResultPrinter::printPlayers(const std::vector<Player> &players, _iobuf *stream) {
for (const auto &player: players) {
fwrite("Full Name:", sizeof(char), 10, stream);
fwrite(player.getlastName().c_str(), sizeof(char), player.getlastName().size(), stream);
fwrite(" ", sizeof(char), 1, stream);
fwrite(player.getfirstName().c_str(), sizeof(char), player.getfirstName().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Gender:", sizeof(char), 7, stream);
fwrite(player.getgender().c_str(), sizeof(char), player.getgender().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Country:", sizeof(char), 8, stream);
fwrite(player.getcountry().c_str(), sizeof(char), player.getcountry().size(), stream);
fwrite("\n-----\n", sizeof(char), 7, stream);
}
}
```
### 6.4 正则表达式
### 6.3 正则表达式
另一个瓶颈是正则表达式的解析,对于C++标准库的正则表达式来说,性能并不是很好,所以这里先使用硬编码的方式处理指令。
......@@ -687,6 +685,59 @@ Command CommandParser::parseCommand(const std::string &input) {
}
```
### 6.4 IO
`ofstream` 是一个比较慢的输出流,所以我们改用内存映射来写文件。
```cpp
void ResultPrinter::writeByteStreamsToFile(const std::string &filename) {
HANDLE hFile = CreateFile(filename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL, nullptr);
if (hFile == INVALID_HANDLE_VALUE) {
LOG_ERROR("Failed to create/open file");
return;
}
DWORD fileSize = 0;
for (const auto &byteStream: byteStreams) {
fileSize += static_cast<DWORD>(byteStream.size);
}
HANDLE hMapFile = CreateFileMapping(hFile, nullptr, PAGE_READWRITE, 0, fileSize, nullptr);
if (hMapFile == nullptr) {
CloseHandle(hFile);
LOG_ERROR("Failed to create file mapping");
return;
}
char *pMappedData = static_cast<char *>(MapViewOfFile(hMapFile, FILE_MAP_ALL_ACCESS, 0, 0, fileSize));
if (pMappedData == nullptr) {
CloseHandle(hMapFile);
CloseHandle(hFile);
LOG_ERROR("Failed to map view of file");
return;
}
char *currentPos = pMappedData;
for (const auto &byteStream: byteStreams) {
memcpy(currentPos, byteStream.data, byteStream.size);
currentPos += byteStream.size;
}
FlushViewOfFile(pMappedData, fileSize);
UnmapViewOfFile(pMappedData);
CloseHandle(hMapFile);
CloseHandle(hFile);
}
```
测试结果如下,使用160w条数据进行测试(速度取决于磁盘速度):
![](/md-files/img_1.png)
## 7. 单元测试
对于核心模块,我在子项目`DWASearch_Tests`编写了一些单元测试。
......
......@@ -9,14 +9,17 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug")
endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ot /Oi /Oy /GL /Gy /GS- /fp:fast /arch:AVX2 /Qpar /Qpar")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ot /Oi /Oy /GL /Gy /GS- /fp:fast /arch:AVX2 /Qpar /favor:INTEL64")
endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_EXE_LINKER_FLAGS "-static")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Ofast -march=native")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -march=native")
endif ()
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND "${CMAKE_CXX_SIMULATE_ID}" STREQUAL "MSVC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /O2 /Ot /Og /Oi /Oy /Gy /GS- /fp:fast /arch:AVX2")
endif ()
add_library(sqlite3 STATIC ${CMAKE_SOURCE_DIR}/lib/sqlite3/sqlite3.c)
include_directories(
......@@ -54,6 +57,7 @@ add_executable(DWASearch DWASearch.cpp
middleware/MatchManager.h
IO/Command.cpp
IO/Command.h
middleware/ByteStream.h
)
target_link_libraries(DWASearch sqlite3)
......
......@@ -10,7 +10,7 @@
#include "MatchManager.h"
namespace Mika {
void initCommandParser(_iobuf *os) {
void initCommandParser() {
static bool initialized = false;
if (!initialized) {
......@@ -18,32 +18,32 @@ namespace Mika {
}
Command::registerCommand("players", [=](const Command &command) {
ResultPrinter::printPlayers(PlayerManager::getAllPlayers(), os);
ResultPrinter::printByteStream(PlayerManager::getAllPlayers());
});
Command::registerCommand("result", [=](const Command &command) {
std::string matchName = command.getArg();
if (matchName.empty()) {
ResultPrinter::printError(os);
ResultPrinter::printError();
return;
}
std::string suffix = command.getSuffix();
if (suffix.empty()) {
auto matches = MatchManager::getMatchResultsByName(matchName);
matches.empty() ? ResultPrinter::printNA(os) : ResultPrinter::printMatchResults(matches, os);
auto byteStream = MatchManager::getMatchResultsByName(matchName);
byteStream.size == 0 ? ResultPrinter::printNA() : ResultPrinter::printByteStream(byteStream);
} else if (suffix == "detail") {
auto matches = MatchManager::getMatchDetailsByName(matchName);
matches.empty() ? ResultPrinter::printNA(os) : ResultPrinter::printMatchResultsDetail(matches, os);
auto byteStream = MatchManager::getMatchDetailsByName(matchName);
byteStream.size == 0 ? ResultPrinter::printNA() : ResultPrinter::printByteStream(byteStream);
}
});
}
void CommandParser::parse(const std::string &command, _iobuf *stream) {
initCommandParser(stream);
void CommandParser::parse(const std::string &command) {
initCommandParser();
if (!Command::executeCommand(parseCommand(command))) {
ResultPrinter::printError(stream);
ResultPrinter::printError();
}
}
......
......@@ -24,7 +24,7 @@ namespace Mika {
static Command parseCommand(const std::string &input);
static void parse(const std::string &command, _iobuf *stream);
static void parse(const std::string &command);
};
} // Mika
......
......@@ -3,68 +3,74 @@
//
#include <iostream>
#include <chrono>
#include "ResultPrinter.h"
namespace Mika {
std::vector<ByteStream> byteStreams;
void ResultPrinter::printPlayers(const std::vector<Player> &players, _iobuf *stream) {
for (const auto &player: players) {
fwrite("Full Name:", sizeof(char), 10, stream);
fwrite(player.getlastName().c_str(), sizeof(char), player.getlastName().size(), stream);
fwrite(" ", sizeof(char), 1, stream);
fwrite(player.getfirstName().c_str(), sizeof(char), player.getfirstName().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Gender:", sizeof(char), 7, stream);
fwrite(player.getgender().c_str(), sizeof(char), player.getgender().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Country:", sizeof(char), 8, stream);
fwrite(player.getcountry().c_str(), sizeof(char), player.getcountry().size(), stream);
fwrite("\n-----\n", sizeof(char), 7, stream);
}
void ResultPrinter::printError() {
static ByteStream error = {"Error\n-----\n", 12};
byteStreams.push_back(error);
}
void ResultPrinter::printMatchResults(const std::vector<MatchResult> &results, _iobuf *stream) {
for (const auto &result: results) {
fwrite("Full Name:", sizeof(char), 10, stream);
fwrite(result.getfullName().c_str(), sizeof(char), result.getfullName().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Rank:", sizeof(char), 5, stream);
auto rank = std::to_string(result.getrank());
fwrite(rank.c_str(), sizeof(char), rank.size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Score:", sizeof(char), 6, stream);
fwrite(result.getscore().c_str(), sizeof(char), result.getscore().size(), stream);
fwrite("\n-----\n", sizeof(char), 7, stream);
}
void ResultPrinter::printNA() {
static ByteStream na = {"N/A\n-----\n", 10};
byteStreams.push_back(na);
}
void ResultPrinter::printMatchResultsDetail(const std::vector<MatchResultDetail> &results, _iobuf *stream) {
for (const auto &result: results) {
fwrite("Full Name:", sizeof(char), 10, stream);
fwrite(result.getfullName().c_str(), sizeof(char), result.getfullName().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Rank:", sizeof(char), 5, stream);
fwrite(result.getrank().c_str(), sizeof(char), result.getrank().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Preliminary Score:", sizeof(char), 18, stream);
fwrite(result.getpreliminaryScore().c_str(), sizeof(char), result.getpreliminaryScore().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Semifinal Score:", sizeof(char), 16, stream);
fwrite(result.getsemifinalScore().c_str(), sizeof(char), result.getsemifinalScore().size(), stream);
fwrite("\n", sizeof(char), 1, stream);
fwrite("Final Score:", sizeof(char), 12, stream);
fwrite(result.getfinalScore().c_str(), sizeof(char), result.getfinalScore().size(), stream);
fwrite("\n-----\n", sizeof(char), 7, stream);
}
void ResultPrinter::printByteStream(ByteStream byteStream) {
byteStreams.emplace_back(byteStream);
}
void ResultPrinter::printError(_iobuf *stream) {
fwrite("Error\n-----\n", sizeof(char), 12, stream);
void ResultPrinter::writeByteStreamsToFile(const std::string &filename) {
HANDLE hFile = CreateFile(filename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL, nullptr);
if (hFile == INVALID_HANDLE_VALUE) {
LOG_ERROR("Failed to create/open file");
return;
}
DWORD fileSize = 0;
for (const auto &byteStream: byteStreams) {
fileSize += static_cast<DWORD>(byteStream.size);
}
HANDLE hMapFile = CreateFileMapping(hFile, nullptr, PAGE_READWRITE, 0, fileSize, nullptr);
if (hMapFile == nullptr) {
CloseHandle(hFile);
LOG_ERROR("Failed to create file mapping");
return;
}
char *pMappedData = static_cast<char *>(MapViewOfFile(hMapFile, FILE_MAP_ALL_ACCESS, 0, 0, fileSize));
if (pMappedData == nullptr) {
CloseHandle(hMapFile);
CloseHandle(hFile);
LOG_ERROR("Failed to map view of file");
return;
}
char *currentPos = pMappedData;
for (const auto &byteStream: byteStreams) {
memcpy(currentPos, byteStream.data, byteStream.size);
currentPos += byteStream.size;
}
FlushViewOfFile(pMappedData, fileSize);
UnmapViewOfFile(pMappedData);
CloseHandle(hMapFile);
CloseHandle(hFile);
}
void ResultPrinter::printNA(_iobuf *stream) {
fwrite("N/A\n-----\n", sizeof(char), 10, stream);
void ResultPrinter::writeByteStreamsToConsole() {
for (const auto &byteStream: byteStreams) {
std::cout.write(byteStream.data, byteStream.size);
}
}
} // Mika
\ No newline at end of file
......@@ -8,6 +8,7 @@
#include <vector>
#include "Player.h"
#include "MatchResult.h"
#include "ByteStream.h"
namespace Mika {
......@@ -21,15 +22,15 @@ namespace Mika {
ResultPrinter &operator=(const ResultPrinter &) = delete;
static void printPlayers(const std::vector<Player> &players, _iobuf *stream = stdout);
static void printByteStream(ByteStream byteStream);
static void printMatchResults(const std::vector<MatchResult> &results, _iobuf *stream = stdout);
static void printError();
static void printMatchResultsDetail(const std::vector<MatchResultDetail> &results, _iobuf *stream = stdout);
static void printNA();
static void printError(_iobuf *stream = stdout);
static void writeByteStreamsToFile(const std::string &filename);
static void printNA(_iobuf *stream = stdout);
static void writeByteStreamsToConsole();
};
} // Mika
......
......@@ -6,10 +6,12 @@
#include <iostream>
#include <fstream>
#include <chrono>
#include <cstring>
#include "UserInput.h"
#include "CommandParser.h"
#include "Logger.h"
#include "ResultPrinter.h"
namespace Mika {
......@@ -17,7 +19,8 @@ namespace Mika {
std::string command;
std::getline(std::cin, command);
CommandParser::parse(command, stdout);
CommandParser::parse(command);
ResultPrinter::writeByteStreamsToConsole();
}
void UserInput::getInput(int argc, char **argv) {
......@@ -30,19 +33,14 @@ namespace Mika {
if (argc == 2) {
for (const auto &command: commands) {
CommandParser::parse(command, stdout);
CommandParser::parse(command);
}
ResultPrinter::writeByteStreamsToConsole();
} else if (argc >= 3) {
FILE *file;
fopen_s(&file, argv[2], "w");
if (file == nullptr) {
LOG_ERROR("File not found");
return;
}
for (const auto &command: commands) {
CommandParser::parse(command, file);
CommandParser::parse(command);
}
fclose(file);
ResultPrinter::writeByteStreamsToFile(argv[2]);
}
} catch (const std::exception &e) {
LOG_ERROR(std::string(e.what()));
......
//
// Created by Adarion on 2024/2/26.
//
#ifndef DWASEARCH_BYTESTREAM_H
#define DWASEARCH_BYTESTREAM_H
#include <cstddef>
namespace Mika {
struct ByteStream {
const char *data;
size_t size;
};
} // Mika
#endif
\ No newline at end of file
......@@ -6,21 +6,49 @@
#include "MatchDAO.h"
namespace Mika {
std::unordered_map<std::string, std::vector<MatchResult>> MatchManager::matchResultsByNames = std::unordered_map<std::string, std::vector<MatchResult>>();
std::unordered_map<std::string, std::vector<MatchResultDetail>> MatchManager::matchDetailsByNames = std::unordered_map<std::string, std::vector<MatchResultDetail>>();
std::unordered_map<std::string, ByteStream> MatchManager::matchResultsByNames =
std::unordered_map < std::string, ByteStream
>();
std::unordered_map<std::string, ByteStream> MatchManager::matchDetailsByNames =
std::unordered_map < std::string, ByteStream
>();
const std::vector<MatchResultDetail> &MatchManager::getMatchDetailsByName(const std::string &name) {
ByteStream MatchManager::getMatchDetailsByName(const std::string &name) {
if (matchDetailsByNames.find(name) == matchDetailsByNames.end()) {
matchDetailsByNames[name] = MatchDAO::getMatchResultsDetail(name);
auto temp = MatchDAO::getMatchResultsDetail(name);
std::string tempString = {};
for (const auto &result: temp) {
tempString += "Full Name:" + result.getfullName() + "\n";
tempString += "Rank:" + result.getrank() + "\n";
tempString += "Preliminary Score:" + result.getpreliminaryScore() + "\n";
tempString += "Semifinal Score:" + result.getsemifinalScore() + "\n";
tempString += "Final Score:" + result.getfinalScore() + "\n";
tempString += "-----\n";
}
char *data = new char[tempString.size()];
memcpy(data, tempString.data(), tempString.size());
matchDetailsByNames[name] = {data, tempString.size()};
}
return matchDetailsByNames[name];
}
const std::vector<MatchResult> &MatchManager::getMatchResultsByName(const std::string &name) {
ByteStream MatchManager::getMatchResultsByName(const std::string &name) {
if (matchResultsByNames.find(name) == matchResultsByNames.end()) {
matchResultsByNames[name] = MatchDAO::getMatchResults(name);
auto temp = MatchDAO::getMatchResults(name);
std::string tempString = {};
for (const auto &result: temp) {
tempString += "Full Name:" + result.getfullName() + "\n";
tempString += "Rank:" + std::to_string(result.getrank()) + "\n";
tempString += "Score:" + result.getscore() + "\n";
tempString += "-----\n";
}
char *data = new char[tempString.size()];
memcpy(data, tempString.data(), tempString.size());
matchResultsByNames[name] = {data, tempString.size()};
}
return matchResultsByNames[name];
......
......@@ -8,13 +8,13 @@
#include <map>
#include "Match.h"
#include "MatchResult.h"
#include "ByteStream.h"
namespace Mika {
class MatchManager {
private:
static std::unordered_map<std::string, std::vector<MatchResult>> matchResultsByNames;
static std::unordered_map<std::string, std::vector<MatchResultDetail>> matchDetailsByNames;
static std::unordered_map<std::string, ByteStream> matchResultsByNames;
static std::unordered_map<std::string, ByteStream> matchDetailsByNames;
public:
MatchManager() = delete;
......@@ -25,9 +25,9 @@ namespace Mika {
MatchManager &operator=(const MatchManager &) = delete;
static const std::vector<MatchResultDetail> &getMatchDetailsByName(const std::string &name);
static ByteStream getMatchDetailsByName(const std::string &name);
static const std::vector<MatchResult> &getMatchResultsByName(const std::string &name);
static ByteStream getMatchResultsByName(const std::string &name);
};
} // Mika
......
......@@ -7,9 +7,9 @@
namespace Mika {
bool PlayerManager::initialized = false;
std::vector<Player> PlayerManager::players;
ByteStream PlayerManager::players;
const std::vector<Player> &PlayerManager::getAllPlayers() {
ByteStream PlayerManager::getAllPlayers() {
if (!initialized) {
initialize();
}
......@@ -17,7 +17,20 @@ namespace Mika {
}
void PlayerManager::initialize() {
players = std::move(PlayerDAO::getAllPlayers());
auto temp = PlayerDAO::getAllPlayers();
std::string tempString = {};
for (const auto &player: temp) {
tempString += "Full Name:" + player.getlastName() + " " + player.getfirstName() + "\n";
tempString += "Gender:" + player.getgender() + "\n";
tempString += "Country:" + player.getcountry() + "\n";
tempString += "-----\n";
}
char *data = new char[tempString.size()];
memcpy(data, tempString.data(), tempString.size());
players = {data, tempString.size()};
initialized = true;
}
......
......@@ -6,11 +6,12 @@
#define DWASEARCH_PLAYERMANAGER_H
#include "Player.h"
#include "ByteStream.h"
namespace Mika {
class PlayerManager {
private:
static std::vector<Player> players;
static ByteStream players;
static bool initialized;
static void initialize();
......@@ -24,7 +25,7 @@ namespace Mika {
PlayerManager &operator=(const PlayerManager &) = delete;
static const std::vector<Player> &getAllPlayers();
static ByteStream getAllPlayers();
};
}
......
......@@ -10,21 +10,38 @@
#ifdef __DEVELOPMENT__
#define DATABASE_FILE "C:\\Users\\Adarion\\CLionProjects\\project-c\\222100414\\src\\data\\diving.db"
#else
// 不知道调用者是否和exe在同一目录,所以用GetModuleFileName获取exe路径
const char *DATABASE_FILE = []() {
static char buf[MAX_PATH];
DWORD length = GetModuleFileName(nullptr, buf, MAX_PATH);
std::string WideStringToUtf8(const std::wstring &wideStr) {
int utf8Length = WideCharToMultiByte(CP_UTF8, 0, wideStr.c_str(), -1, nullptr, 0, nullptr, nullptr);
if (utf8Length == 0) {
LOG_ERROR("WideCharToMultiByte failed");
return "";
}
std::string utf8Str(utf8Length, '\0');
WideCharToMultiByte(CP_UTF8, 0, wideStr.c_str(), -1, &utf8Str[0], utf8Length, nullptr, nullptr);
return utf8Str;
}
// 不知道调用者是否和exe在同一目录,所以用GetModuleFileNameW获取exe路径
const char *DATABASE_FILE = []() -> const char * {
static wchar_t buf[MAX_PATH];
DWORD length = GetModuleFileNameW(nullptr, buf, MAX_PATH);
if (length == 0) {
LOG_ERROR("GetModuleFileName failed");
return "";
}
static std::string path = buf;
auto pos = path.find_last_of('\\');
static std::wstring path = buf;
auto pos = path.find_last_of(L'\\');
path = path.substr(0, pos);
path += R"(\src\data\diving.db)";
return path.c_str();
path += L"\\src\\data\\diving.db";
// 要命的中文路径
static std::string utf8Path = WideStringToUtf8(path);
return utf8Path.c_str();
}();
#endif
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册