Implement method to verify and write torrent files and file manifests to disk.
Also add some helper methods to file manifest and torrent file get add useful info encapsulated
in the name. Lastly fixed an apart bug in load_directory helper related to files from
boost::directory_iterator being in an arbitrary rather than lexicographical order.
Change-Id: I8a2c2f3224caec65cf97a418785778234a0d4d73
diff --git a/src/file-manifest.hpp b/src/file-manifest.hpp
index 8c4680a..8366db2 100644
--- a/src/file-manifest.hpp
+++ b/src/file-manifest.hpp
@@ -141,6 +141,14 @@
data_packet_size() const;
/// Returns the 'data_packet_size' of this FileManifest
+ size_t
+ submanifest_number() const;
+ /// Return the submanifest number for this FileManifest
+
+ std::string
+ file_name() const;
+ /// Return the file name for this FileManifest
+
std::shared_ptr<Name>
submanifest_ptr() const;
/// Returns the 'submanifest_ptr' of this FileManifest, or 'nullptr' is none exists
@@ -301,6 +309,19 @@
return m_submanifestPtr;
}
+inline std::string
+FileManifest::file_name() const
+{
+ return name().getSubName(1, name().size() - 2).toUri();
+}
+
+
+inline size_t
+FileManifest::submanifest_number() const
+{
+ return name().get(name().size() - 1).toSequenceNumber();
+}
+
inline void
FileManifest::set_submanifest_ptr(std::shared_ptr<Name> subManifestPtr)
{
diff --git a/src/torrent-file.hpp b/src/torrent-file.hpp
index 98e222c..0536c17 100644
--- a/src/torrent-file.hpp
+++ b/src/torrent-file.hpp
@@ -107,6 +107,18 @@
getCatalog() const;
/**
+ * @brief Get the segment number for this torrent file
+ */
+ size_t
+ getSegmentNumber() const;
+
+ /**
+ * @brief Get the directory name for this torrent file
+ */
+ std::string
+ getTorrentFilePath() const;
+
+ /**
* @brief Decode from wire format
*/
void
@@ -272,6 +284,19 @@
return m_commonPrefix;
}
+inline std::string
+TorrentFile::getTorrentFilePath() const
+{
+ return (0 == getSegmentNumber() ? getFullName().get(-3) : getFullName().get(-4)).toUri();
+}
+
+inline size_t
+TorrentFile::getSegmentNumber() const
+{
+ const auto& lastComponent = getName().get(getName().size() - 1);
+ return lastComponent.isSequenceNumber() ? lastComponent.toSequenceNumber() : 0;
+}
+
} // namespace ntorrent
diff --git a/src/torrent-manager.cpp b/src/torrent-manager.cpp
index f62491f..afda819 100644
--- a/src/torrent-manager.cpp
+++ b/src/torrent-manager.cpp
@@ -28,13 +28,16 @@
load_directory(const string& dirPath,
ndn::io::IoEncoding encoding = ndn::io::IoEncoding::BASE_64) {
vector<T> structures;
-
+ std::set<string> fileNames;
if (fs::exists(dirPath)) {
- for(fs::directory_iterator it(dirPath);
- it != fs::directory_iterator();
+ for(fs::recursive_directory_iterator it(dirPath);
+ it != fs::recursive_directory_iterator();
++it)
{
- auto data_ptr = ndn::io::load<T>(it->path().string(), encoding);
+ fileNames.insert(it->path().string());
+ }
+ for (const auto& f : fileNames) {
+ auto data_ptr = ndn::io::load<T>(f, encoding);
if (nullptr != data_ptr) {
structures.push_back(*data_ptr);
}
@@ -130,6 +133,7 @@
security::KeyChain key_chain;
Name currSegmentFullName = initialSegmentName;
vector<TorrentFile> torrentSegments = load_directory<TorrentFile>(torrentFilePath);
+
// Starting with the initial segment name, verify the names, loading next name from torrentSegment
for (auto it = torrentSegments.begin(); it != torrentSegments.end(); ++it) {
TorrentFile& segment = *it;
@@ -218,7 +222,7 @@
const TorrentFile& torrentFile)
{
vector<Data> packets;
- auto subManifestNum = manifest.name().get(manifest.name().size() - 1).toSequenceNumber();
+ auto subManifestNum = manifest.submanifest_number();
packets = packetize_file(filePath,
manifest.name(),
@@ -243,8 +247,7 @@
const FileManifest& manifest)
{
// construct the file name
- const auto manifestName = manifest.name();
- auto fileName = manifestName.getSubName(1, manifestName.size() - 2).toUri();
+ auto fileName = manifest.file_name();
auto filePath = dataPath + fileName;
vector<bool> fileBitMap(manifest.catalog().size());
auto fbits = fs::fstream::out | fs::fstream::binary;
@@ -283,7 +286,7 @@
currCatalog = currTorrentFile_it->getCatalog();
}
// construct the file name
- auto fileName = m.name().getSubName(1, m.name().size() - 2).toUri();
+ auto fileName = m.file_name();
fs::path filePath = m_dataPath + fileName;
// If there are any valid packets, add corresponding state to manager
if (!fs::exists(filePath)) {
@@ -333,7 +336,7 @@
if (nullptr == fileState.first) {
fileState = initializeFileState(m_dataPath, *manifest_it);
}
- auto packetNum = packetName.get(packet.getName().size() - 1).toSequenceNumber();
+ auto packetNum = packetName.get(packetName.size() - 1).toSequenceNumber();
// if we already have the packet, do not rewrite it.
if (fileState.second[packetNum]) {
return false;
@@ -355,6 +358,78 @@
return true;
}
+bool TorrentManager::writeTorrentSegment(const TorrentFile& segment, const std::string& path)
+{
+ // validate that this torrent segment belongs to our torrent
+ auto torrentPrefix = m_torrentFileName.getSubName(0, m_torrentFileName.size() - 1);
+ if (!torrentPrefix.isPrefixOf(segment.getName())) {
+ return false;
+ }
+
+ auto segmentNum = segment.getSegmentNumber();
+ // check if we already have it
+ if (m_torrentSegments.end() != std::find(m_torrentSegments.begin(), m_torrentSegments.end(),
+ segment))
+ {
+ return false;
+ }
+ // write to disk at path
+ if (!fs::exists(path)) {
+ fs::create_directories(path);
+ }
+ auto filename = path + to_string(segmentNum);
+ // if there is already a file on disk for this torrent segment, determine if we should override
+ if (fs::exists(filename)) {
+ auto segmentOnDisk_ptr = io::load<TorrentFile>(filename);
+ if (nullptr != segmentOnDisk_ptr && *segmentOnDisk_ptr == segment) {
+ return false;
+ }
+ }
+ io::save(segment, filename);
+ // add to collection
+ auto it = std::find_if(m_torrentSegments.begin(), m_torrentSegments.end(),
+ [segmentNum](const TorrentFile& t){
+ return t.getSegmentNumber() > segmentNum;
+ });
+ m_torrentSegments.insert(it, segment);
+ return true;
+}
+
+bool TorrentManager::writeFileManifest(const FileManifest& manifest, const std::string& path)
+{
+ auto subManifestNum = manifest.submanifest_number();
+ fs::path filename = path + manifest.file_name() + "/" + to_string(subManifestNum);
+ // check if we already have it
+ if (m_fileManifests.end() != std::find(m_fileManifests.begin(), m_fileManifests.end(),
+ manifest))
+ {
+ return false;
+ }
+
+ // write to disk at path
+ if (!fs::exists(filename.parent_path())) {
+ boost::filesystem::create_directories(filename.parent_path());
+ }
+ // if there is already a file on disk for this torrent segment, determine if we should override
+ if (fs::exists(filename)) {
+ auto submanifestOnDisk_ptr = io::load<FileManifest>(filename.string());
+ if (nullptr != submanifestOnDisk_ptr && *submanifestOnDisk_ptr == manifest) {
+ return false;
+ }
+ }
+ io::save(manifest, filename.string());
+ // add to collection
+ // add to collection
+ auto it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
+ [&manifest](const FileManifest& m){
+ return m.file_name() > manifest.file_name()
+ || (m.file_name() == manifest.file_name()
+ && (m.submanifest_number() > manifest.submanifest_number()));
+ });
+ m_fileManifests.insert(it, manifest);
+ return true;
+}
+
void TorrentManager::seed(const Data& data) const {
// TODO(msweatt) IMPLEMENT ME
}
diff --git a/src/torrent-manager.hpp b/src/torrent-manager.hpp
index ca83a2b..fc790a8 100644
--- a/src/torrent-manager.hpp
+++ b/src/torrent-manager.hpp
@@ -28,7 +28,6 @@
#include <ndn-cxx/data.hpp>
#include <boost/filesystem/fstream.hpp>
-#include <boost/utility.hpp>
#include <functional>
#include <memory>
@@ -41,7 +40,7 @@
namespace ndn {
namespace ntorrent {
-class TorrentManager : boost::noncopyable {
+class TorrentManager : noncopyable {
/**
* \class TorrentManager
*
@@ -93,9 +92,34 @@
protected:
bool writeData(const Data& packet);
- // Write the Data packet to disk, return 'true' if data successfully written to disk 'false'
- // otherwise. Behavior is undefined unless the corresponding file manifest has already been
- // downloaded.
+ /*
+ * \brief Write @p packet composed of torrent date to disk.
+ * @param packet The data packet to be written to the disk
+ * Write the Data packet to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless the corresponding file manifest has already been
+ * downloaded.
+ */
+
+ bool writeTorrentSegment(const TorrentFile& segment, const std::string& path);
+ /*
+ * \brief Write the @p segment torrent segment to disk at the specified path.
+ * @param segment The torrent file segment to be written to disk
+ * @param path The path at which to write the torrent file segment
+ * Write the segment to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless @segment is a correct segment for the torrent file of
+ * this manager and @p path is the directory used for all segments of this torrent file.
+ */
+
+ bool writeFileManifest(const FileManifest& manifest, const std::string& path);
+ /*
+ * \brief Write the @p manifest file manifest to disk at the specified @p path.
+ * @param manifest The file manifest to be written to disk
+ * @param path The path at which to write the file manifest
+ * Write the file manifest to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless @manifest is a correct file manifest for a file in the
+ * torrent file of this manager and @p path is the directory used for all file manifests of this
+ * torrent file.
+ */
void onDataReceived(const Data& data);
diff --git a/tests/unit-tests/torrent-manager.t.cpp b/tests/unit-tests/torrent-manager.t.cpp
index 910704d..722173b 100644
--- a/tests/unit-tests/torrent-manager.t.cpp
+++ b/tests/unit-tests/torrent-manager.t.cpp
@@ -65,6 +65,13 @@
bool writeData(const Data& data) {
return TorrentManager::writeData(data);
}
+
+ bool writeTorrentSegment(const TorrentFile& segment, const std::string& path) {
+ return TorrentManager::writeTorrentSegment(segment, path);
+ }
+ bool writeFileManifest(const FileManifest& manifest, const std::string& path) {
+ return TorrentManager::writeFileManifest(manifest, path);
+ }
};
BOOST_AUTO_TEST_SUITE(TestTorrentManagerInitialize)
@@ -98,13 +105,13 @@
auto filename = torrentPath + to_string(fileNum);
io::save(t, filename);
}
- fileNum = 0;
+ //fileNum = 0;
auto manifestPath = dirPath + "manifests/";
boost::filesystem::create_directory(manifestPath);
for (const auto& m : manifests) {
- fileNum++;
- auto filename = manifestPath + to_string(fileNum);
- io::save(m, filename);
+ fs::path filename = manifestPath + m.file_name() + "/" + to_string(m.submanifest_number());
+ boost::filesystem::create_directories(filename.parent_path());
+ io::save(m, filename.string());
}
// Initialize and verify
TestTorrentManager manager("/NTORRENT/foo/torrent-file/sha256digest=02c737fd4c6e7de4b4825b089f39700c2dfa8fd2bb2b91f09201e357c4463253",
@@ -200,13 +207,12 @@
auto filename = torrentPath + to_string(fileNum);
io::save(t, filename);
}
- fileNum = 0;
auto manifestPath = dirPath + "manifests/";
boost::filesystem::create_directory(manifestPath);
for (const auto& m : manifests) {
- fileNum++;
- auto filename = manifestPath + to_string(fileNum);
- io::save(m, filename);
+ fs::path filename = manifestPath + m.file_name() + to_string(m.submanifest_number());
+ boost::filesystem::create_directory(filename.parent_path());
+ io::save(m, filename.string());
}
// Initialize and verify
TestTorrentManager manager("/NTORRENT/foo/torrent-file/sha256digest=02c737fd4c6e7de4b4825b089f39700c2dfa8fd2bb2b91f09201e357c4463253",
@@ -263,13 +269,13 @@
auto filename = torrentPath + to_string(fileNum);
io::save(t, filename);
}
- fileNum = 0;
+ //fileNum = 0;
auto manifestPath = dirPath + "manifests/";
boost::filesystem::create_directory(manifestPath);
for (const auto& m : manifests) {
- fileNum++;
- auto filename = manifestPath + to_string(fileNum);
- io::save(m, filename);
+ fs::path filename = manifestPath + m.file_name() + to_string(m.submanifest_number());
+ boost::filesystem::create_directory(filename.parent_path());
+ io::save(m, filename.string());
}
// Initialize manager
TestTorrentManager manager("/NTORRENT/foo/torrent-file/sha256digest=02c737fd4c6e7de4b4825b089f39700c2dfa8fd2bb2b91f09201e357c4463253",
@@ -322,6 +328,150 @@
fs::remove_all(".appdata");
}
+BOOST_AUTO_TEST_CASE(CheckWriteTorrentComplete)
+{
+ const struct {
+ const char *d_directoryPath;
+ const char *d_initialSegmentName;
+ size_t d_namesPerSegment;
+ size_t d_subManifestSize;
+ size_t d_dataPacketSize;
+ } DATA [] = {
+ {"tests/testdata/foo", "/NTORRENT/foo/torrent-file/sha256digest=02c737fd4c6e7de4b4825b089f39700c2dfa8fd2bb2b91f09201e357c4463253", 1024, 1024, 1024},
+ {"tests/testdata/foo", "/NTORRENT/foo/torrent-file/sha256digest=96d900d6788465f9a7b00191581b004c910d74b3762d141ec0e82173731bc9f4", 1, 1, 1024},
+ };
+ enum { NUM_DATA = sizeof DATA / sizeof *DATA };
+ for (int i = 0; i < NUM_DATA; ++i) {
+ auto directoryPath = DATA[i].d_directoryPath;
+ Name initialSegmentName = DATA[i].d_initialSegmentName;
+ auto namesPerSegment = DATA[i].d_namesPerSegment;
+ auto dataPacketSize = DATA[i].d_dataPacketSize;
+ auto subManifestSize = DATA[i].d_subManifestSize;
+
+ vector<TorrentFile> torrentSegments;
+ std::string filePath = "tests/testdata/temp";
+ // get torrent files
+ {
+ auto temp = TorrentFile::generate(directoryPath,
+ namesPerSegment,
+ subManifestSize,
+ dataPacketSize,
+ false);
+ torrentSegments = temp.first;
+ }
+ // Initialize manager
+ TestTorrentManager manager(initialSegmentName,
+ filePath);
+ manager.Initialize();
+ std::string dirPath = ".appdata/foo/";
+ std::string torrentPath = dirPath + "torrent_files/";
+ BOOST_CHECK(manager.torrentSegments().empty());
+ for (const auto& t : torrentSegments) {
+ BOOST_CHECK(manager.writeTorrentSegment(t, torrentPath));
+ }
+ BOOST_CHECK(manager.torrentSegments() == torrentSegments);
+ // check that initializing a new manager also gets all the torrent torrentSegments
+ TestTorrentManager manager2(initialSegmentName,
+ filePath);
+ manager2.Initialize();
+ BOOST_CHECK(manager2.torrentSegments() == torrentSegments);
+
+ // start anew
+ fs::remove_all(torrentPath);
+ fs::create_directories(torrentPath);
+ manager.Initialize();
+ BOOST_CHECK(manager.torrentSegments().empty());
+
+ // check that there is no dependence on the order of torrent segments
+ // randomize the order of the torrent segments
+ auto torrentSegmentsRandom = torrentSegments;
+ std::random_shuffle(torrentSegmentsRandom.begin(), torrentSegmentsRandom.end());
+ for (const auto& t : torrentSegmentsRandom) {
+ BOOST_CHECK(manager.writeTorrentSegment(t, torrentPath));
+ }
+ BOOST_CHECK(manager.torrentSegments() == torrentSegments);
+ fs::remove_all(".appdata");
+ }
+}
+
+BOOST_AUTO_TEST_CASE(CheckWriteManifestComplete)
+{
+ std::string dirPath = ".appdata/foo/";
+ std::string torrentPath = dirPath + "torrent_files/";
+ std::string manifestPath = dirPath + "manifests/";
+
+ const struct {
+ const char *d_directoryPath;
+ const char *d_initialSegmentName;
+ size_t d_namesPerSegment;
+ size_t d_subManifestSize;
+ size_t d_dataPacketSize;
+ } DATA [] = {
+ {"tests/testdata/foo", "/NTORRENT/foo/torrent-file/sha256digest=02c737fd4c6e7de4b4825b089f39700c2dfa8fd2bb2b91f09201e357c4463253", 1024, 1024, 1024},
+ {"tests/testdata/foo", "/NTORRENT/foo/torrent-file/sha256digest=02c737fd4c6e7de4b4825b089f39700c2dfa8fd2bb2b91f09201e357c4463253", 128, 128, 1024},
+ };
+ enum { NUM_DATA = sizeof DATA / sizeof *DATA };
+ for (int i = 0; i < NUM_DATA; ++i) {
+ auto directoryPath = DATA[i].d_directoryPath;
+ Name initialSegmentName = DATA[i].d_initialSegmentName;
+ auto namesPerSegment = DATA[i].d_namesPerSegment;
+ auto dataPacketSize = DATA[i].d_dataPacketSize;
+ auto subManifestSize = DATA[i].d_subManifestSize;
+
+ vector<FileManifest> manifests;
+ vector<TorrentFile> torrentSegments;
+
+ std::string filePath = "tests/testdata/temp";
+ // get torrent files and manifests
+ {
+ auto temp = TorrentFile::generate(directoryPath,
+ namesPerSegment,
+ subManifestSize,
+ dataPacketSize,
+ false);
+ torrentSegments = temp.first;
+ auto temp1 = temp.second;
+ for (const auto& ms : temp1) {
+ manifests.insert(manifests.end(), ms.first.begin(), ms.first.end());
+ }
+ }
+ TestTorrentManager manager(initialSegmentName,
+ filePath);
+ manager.Initialize();
+ for (const auto& t : torrentSegments) {
+ manager.writeTorrentSegment(t, torrentPath);
+ }
+
+ BOOST_CHECK(manager.fileManifests().empty());
+ for (const auto& m : manifests) {
+ BOOST_CHECK(manager.writeFileManifest(m, manifestPath));
+ }
+ BOOST_CHECK(manager.fileManifests() == manifests);
+
+ TestTorrentManager manager2(initialSegmentName,
+ filePath);
+
+ manager2.Initialize();
+ BOOST_CHECK(manager2.fileManifests() == manifests);
+
+ // start anew
+ fs::remove_all(manifestPath);
+ fs::create_directories(manifestPath);
+ manager.Initialize();
+ BOOST_CHECK(manager.fileManifests().empty());
+
+ // check that there is no dependence on the order of torrent segments
+ // randomize the order of the torrent segments
+ auto fileManifestsRandom = manifests;
+ std::random_shuffle(fileManifestsRandom.begin(), fileManifestsRandom.end());
+ for (const auto& m : fileManifestsRandom) {
+ BOOST_CHECK(manager.writeFileManifest(m, manifestPath));
+ }
+ BOOST_CHECK(manager2.fileManifests() == manifests);
+ fs::remove_all(".appdata");
+ }
+}
+
BOOST_AUTO_TEST_SUITE_END()