Implement a method in TorrentManger to write a Data packet of a torrent to the correct location on
disk. Also create a utility method to initialize the structure for fileState, fixing a bug of
opening a non-existent file with the fs::stream::in bit.
Change-Id: Ife08c2fef93d6a6cba906c3735ffac57506b7750
diff --git a/src/torrent-manager.cpp b/src/torrent-manager.cpp
index 305fa3b..f62491f 100644
--- a/src/torrent-manager.cpp
+++ b/src/torrent-manager.cpp
@@ -238,6 +238,25 @@
return packets;
}
+static std::pair<std::shared_ptr<fs::fstream>, std::vector<bool>>
+initializeFileState(const string& dataPath,
+ const FileManifest& manifest)
+{
+ // construct the file name
+ const auto manifestName = manifest.name();
+ auto fileName = manifestName.getSubName(1, manifestName.size() - 2).toUri();
+ auto filePath = dataPath + fileName;
+ vector<bool> fileBitMap(manifest.catalog().size());
+ auto fbits = fs::fstream::out | fs::fstream::binary;
+ // if file exists, use in O/W use concatenate mode
+ fbits |= fs::exists(filePath) ? fs::fstream::in : fs::fstream::ate;
+ auto s = std::make_shared<fs::fstream>(filePath, fbits);
+ if (!*s) {
+ BOOST_THROW_EXCEPTION(io::Error("Cannot open: " + dataPath));
+ }
+ return std::make_pair(s, fileBitMap);
+}
+
void TorrentManager::Initialize()
{
// .../<torrent_name>/torrent-file/<implicit_digest>
@@ -265,29 +284,28 @@
}
// construct the file name
auto fileName = m.name().getSubName(1, m.name().size() - 2).toUri();
- auto filePath = m_dataPath + fileName;
+ fs::path filePath = m_dataPath + fileName;
// If there are any valid packets, add corresponding state to manager
- auto packets = intializeDataPackets(filePath, m, *currTorrentFile_it);
+ if (!fs::exists(filePath)) {
+ boost::filesystem::create_directories(filePath.parent_path());
+ continue;
+ }
+ auto packets = intializeDataPackets(filePath.string(), m, *currTorrentFile_it);
if (!packets.empty()) {
- // build the bit map
- auto catalog = m.catalog();
- vector<bool> fileBitMap(catalog.size());
+ m_fileStates[m.getFullName()] = initializeFileState(m_dataPath, m);
+ auto& fileBitMap = m_fileStates[m.getFullName()].second;
auto read_it = packets.begin();
size_t i = 0;
- for (auto name : catalog) {
+ for (auto name : m.catalog()) {
if (name == read_it->getFullName()) {
++read_it;
- fileBitMap[i]= true;
+ fileBitMap[i] = true;
}
++i;
}
for (const auto& d : packets) {
seed(d);
}
- auto s = std::make_shared<fs::fstream>(filePath, fs::fstream::binary
- | fs::fstream::in
- | fs::fstream::out);
- m_fileStates[m.getFullName()] = std::make_pair(s, fileBitMap);
}
}
for (const auto& t : m_torrentSegments) {
@@ -296,7 +314,45 @@
for (const auto& m : m_fileManifests) {
seed(m);
}
+}
+bool TorrentManager::writeData(const Data& packet)
+{
+ // find correct manifest
+ const auto& packetName = packet.getName();
+ auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
+ [&packetName](const FileManifest& m) {
+ return m.getName().isPrefixOf(packetName);
+ });
+ if (m_fileManifests.end() == manifest_it) {
+ return false;
+ }
+ // get file state out
+ auto& fileState = m_fileStates[manifest_it->getFullName()];
+ // if there is no open stream to the file
+ if (nullptr == fileState.first) {
+ fileState = initializeFileState(m_dataPath, *manifest_it);
+ }
+ auto packetNum = packetName.get(packet.getName().size() - 1).toSequenceNumber();
+ // if we already have the packet, do not rewrite it.
+ if (fileState.second[packetNum]) {
+ return false;
+ }
+ auto packetOffset = packetNum * manifest_it->data_packet_size();
+ // write data to disk
+ fileState.first->seekg(packetOffset);
+ try {
+ auto content = packet.getContent();
+ std::vector<char> data(content.value_begin(), content.value_end());
+ fileState.first->write(&data[0], data.size());
+ }
+ catch (io::Error &e) {
+ std::cerr << e.what() << std::endl;
+ return false;
+ }
+ // update bitmap
+ fileState.second[packetNum] = true;
+ return true;
}
void TorrentManager::seed(const Data& data) const {
diff --git a/src/torrent-manager.hpp b/src/torrent-manager.hpp
index 3c4b923..ca83a2b 100644
--- a/src/torrent-manager.hpp
+++ b/src/torrent-manager.hpp
@@ -92,6 +92,11 @@
// Seed the specified 'data' to the network.
protected:
+ bool writeData(const Data& packet);
+ // Write the Data packet to disk, return 'true' if data successfully written to disk 'false'
+ // otherwise. Behavior is undefined unless the corresponding file manifest has already been
+ // downloaded.
+
void onDataReceived(const Data& data);
void onInterestReceived(const Name& name);
diff --git a/tests/unit-tests/torrent-manager.t.cpp b/tests/unit-tests/torrent-manager.t.cpp
index 59392aa..910704d 100644
--- a/tests/unit-tests/torrent-manager.t.cpp
+++ b/tests/unit-tests/torrent-manager.t.cpp
@@ -24,6 +24,8 @@
#include "torrent-manager.hpp"
#include "torrent-file.hpp"
+#include <set>
+
#include <boost/filesystem.hpp>
#include <ndn-cxx/util/io.hpp>
@@ -53,8 +55,16 @@
}
std::vector<bool> fileState(const ndn::Name& manifestName) {
+ auto fout = m_fileStates[manifestName].first;
+ if (nullptr != fout) {
+ fout->flush();
+ }
return m_fileStates[manifestName].second;
}
+
+ bool writeData(const Data& data) {
+ return TorrentManager::writeData(data);
+ }
};
BOOST_AUTO_TEST_SUITE(TestTorrentManagerInitialize)
@@ -74,9 +84,7 @@
torrentSegments = temp.first;
auto temp1 = temp.second;
for (const auto& ms : temp1) {
- for (const auto& m : ms.first) {
- manifests.push_back(m);
- }
+ manifests.insert(manifests.end(), ms.first.begin(), ms.first.end());
}
}
// write the torrent segments and manifests to disk
@@ -177,10 +185,8 @@
torrentSegments = temp.first;
auto temp1 = temp.second;
temp1.pop_back(); // remove the manifests for the last file
- for (const auto& ms : temp1) {
- for (const auto& m : ms.first) {
- manifests.push_back(m);
- }
+ for (const auto& ms : temp1) {
+ manifests.insert(manifests.end(), ms.first.begin(), ms.first.end());
}
}
// write the torrent segments and manifests to disk
@@ -193,7 +199,6 @@
fileNum++;
auto filename = torrentPath + to_string(fileNum);
io::save(t, filename);
-
}
fileNum = 0;
auto manifestPath = dirPath + "manifests/";
@@ -224,6 +229,102 @@
BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE(CheckTorrentManagerUtilities)
+
+BOOST_AUTO_TEST_CASE(CheckWriteDataComplete)
+{
+ vector<FileManifest> manifests;
+ vector<TorrentFile> torrentSegments;
+ // for each file, the data packets
+ std::vector<vector<Data>> fileData;
+ std::string filePath = "tests/testdata/temp";
+ // get torrent files and manifests
+ {
+ auto temp = TorrentFile::generate("tests/testdata/foo",
+ 1024,
+ 1024,
+ 1024,
+ true);
+ torrentSegments = temp.first;
+ auto temp1 = temp.second;
+ for (const auto& ms : temp1) {
+ manifests.insert(manifests.end(), ms.first.begin(), ms.first.end());
+ fileData.push_back(ms.second);
+ }
+ }
+ // write the torrent segments and manifests to disk
+ std::string dirPath = ".appdata/foo/";
+ boost::filesystem::create_directories(dirPath);
+ std::string torrentPath = dirPath + "torrent_files/";
+ boost::filesystem::create_directories(torrentPath);
+ auto fileNum = 0;
+ for (const auto& t : torrentSegments) {
+ fileNum++;
+ auto filename = torrentPath + to_string(fileNum);
+ io::save(t, filename);
+ }
+ fileNum = 0;
+ auto manifestPath = dirPath + "manifests/";
+ boost::filesystem::create_directory(manifestPath);
+ for (const auto& m : manifests) {
+ fileNum++;
+ auto filename = manifestPath + to_string(fileNum);
+ io::save(m, filename);
+ }
+ // Initialize manager
+ TestTorrentManager manager("/NTORRENT/foo/torrent-file/sha256digest=02c737fd4c6e7de4b4825b089f39700c2dfa8fd2bb2b91f09201e357c4463253",
+ filePath);
+ manager.Initialize();
+ // check that initially there is no data on disk
+ for (auto m : manager.fileManifests()) {
+ auto fileState = manager.fileState(m.getFullName());
+ BOOST_CHECK(fileState.empty());
+ }
+ // write all data to disk (for each file manifest)
+ auto manifest_it = manifests.begin();
+ for (auto& data : fileData) {
+ for (auto& d : data) {
+ BOOST_CHECK(manager.writeData(d));
+ }
+ // check that the state is updated appropriately
+ auto fileState = manager.fileState(manifest_it->getFullName());
+ for (auto s : fileState) {
+ BOOST_CHECK(s);
+ }
+ ++manifest_it;
+ }
+ // get the file names (ascending)
+ std::set<std::string> fileNames;
+ for (auto i = fs::recursive_directory_iterator(filePath + "/foo");
+ i != fs::recursive_directory_iterator();
+ ++i) {
+ fileNames.insert(i->path().string());
+ }
+ // verify file by file that the data packets are written correctly
+ auto f_it = fileData.begin();
+ for (auto f : fileNames) {
+ // read file from disk
+ std::vector<uint8_t> file_bytes;
+ fs::ifstream is(f, fs::ifstream::binary | fs::ifstream::in);
+ is >> std::noskipws;
+ std::istream_iterator<uint8_t> start(is), end;
+ file_bytes.insert(file_bytes.end(), start, end);
+ std::vector<uint8_t> data_bytes;
+ // get content from data packets
+ for (const auto& d : *f_it) {
+ auto content = d.getContent();
+ data_bytes.insert(data_bytes.end(), content.value_begin(), content.value_end());
+ }
+ BOOST_CHECK(data_bytes == file_bytes);
+ ++f_it;
+ }
+ fs::remove_all(filePath);
+ fs::remove_all(".appdata");
+}
+
+
+BOOST_AUTO_TEST_SUITE_END()
+
} // namespace tests
} // namespace nTorrent
} // namespace ndn
\ No newline at end of file