Abstracted out some repeated I/O helpers into a IoUtil and updated affected components.
Change-Id: I151b880b29c526cb1ccb9717fabc63d1842cd1c1
diff --git a/src/file-manifest.cpp b/src/file-manifest.cpp
index ca17148..2e6dcd2 100644
--- a/src/file-manifest.cpp
+++ b/src/file-manifest.cpp
@@ -20,6 +20,8 @@
*/
#include "file-manifest.hpp"
+#include "util/io-util.hpp"
+
#include <limits>
#include <boost/assert.hpp>
@@ -71,80 +73,6 @@
return manifestName;
}
-static std::vector<Data>
-packetize_file(const fs::path& filePath,
- const ndn::Name& commonPrefix,
- size_t dataPacketSize,
- size_t subManifestSize,
- size_t subManifestNum)
-{
- BOOST_ASSERT(0 < dataPacketSize);
- size_t APPROX_BUFFER_SIZE = std::numeric_limits<int>::max(); // 2 * 1024 * 1024 *1024
- auto file_size = fs::file_size(filePath);
- auto start_offset = subManifestNum * subManifestSize * dataPacketSize;
- // determine the number of bytes in this submanifest
- auto subManifestLength = subManifestSize * dataPacketSize;
- auto remainingFileLength = file_size - start_offset;
- subManifestLength = remainingFileLength < subManifestLength
- ? remainingFileLength
- : subManifestLength;
- std::vector<Data> packets;
- packets.reserve(subManifestLength/dataPacketSize + 1);
- fs::ifstream fs(filePath, fs::ifstream::binary);
- if (!fs) {
- BOOST_THROW_EXCEPTION(FileManifest::Error("IO Error when opening" + filePath.string()));
- }
- // ensure that buffer is large enough to contain whole packets
- // buffer size is either the entire file or the smallest number of data packets >= 2 GB
- auto buffer_size =
- subManifestLength < APPROX_BUFFER_SIZE ?
- subManifestLength :
- APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
- APPROX_BUFFER_SIZE :
- APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
- std::vector<char> file_bytes;
- file_bytes.reserve(buffer_size);
- size_t bytes_read = 0;
- fs.seekg(start_offset);
- while(fs && bytes_read < subManifestLength && !fs.eof()) {
- // read the file into the buffer
- fs.read(&file_bytes.front(), buffer_size);
- auto read_size = fs.gcount();
- if (fs.bad() || read_size < 0) {
- BOOST_THROW_EXCEPTION(FileManifest::Error("IO Error when reading" + filePath.string()));
- }
- bytes_read += read_size;
- char *curr_start = &file_bytes.front();
- for (size_t i = 0u; i < buffer_size; i += dataPacketSize) {
- // Build a packet from the data
- Name packetName = commonPrefix;
- packetName.appendSequenceNumber(packets.size());
- Data d(packetName);
- auto content_length = i + dataPacketSize > buffer_size ? buffer_size - i : dataPacketSize;
- d.setContent(encoding::makeBinaryBlock(tlv::Content, curr_start, content_length));
- curr_start += content_length;
- // append to the collection
- packets.push_back(d);
- }
- file_bytes.clear();
- // recompute the buffer_size
- buffer_size =
- subManifestLength - bytes_read < APPROX_BUFFER_SIZE ?
- subManifestLength - bytes_read :
- APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
- APPROX_BUFFER_SIZE :
- APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
- }
- fs.close();
- packets.shrink_to_fit();
- security::KeyChain key_chain;
- // sign all the packets
- for (auto& p : packets) {
- key_chain.sign(p, signingWithSha256());
- }
- return packets;
-}
-
// CLASS METHODS
std::pair<std::vector<FileManifest>, std::vector<Data>>
FileManifest::generate(const std::string& filePath,
@@ -176,11 +104,11 @@
// append the packet number
curr_manifest_name.appendSequenceNumber(manifests.size());
FileManifest curr_manifest(curr_manifest_name, dataPacketSize, manifestPrefix);
- auto packets = packetize_file(path,
- curr_manifest_name,
- dataPacketSize,
- subManifestSize,
- subManifestNum);
+ auto packets = IoUtil::packetize_file(path,
+ curr_manifest_name,
+ dataPacketSize,
+ subManifestSize,
+ subManifestNum);
if (returnData) {
allPackets.insert(allPackets.end(), packets.begin(), packets.end());
}
diff --git a/src/torrent-manager.cpp b/src/torrent-manager.cpp
index 155d7cf..cb90908 100644
--- a/src/torrent-manager.cpp
+++ b/src/torrent-manager.cpp
@@ -2,6 +2,7 @@
#include "file-manifest.hpp"
#include "torrent-file.hpp"
+#include "util/io-util.hpp"
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
@@ -21,153 +22,16 @@
using std::string;
using std::vector;
-namespace {
-// TODO(msweatt) Move this to a utility
-template<typename T>
-static vector<T>
-load_directory(const string& dirPath,
- ndn::io::IoEncoding encoding = ndn::io::IoEncoding::BASE_64) {
- vector<T> structures;
- std::set<string> fileNames;
- if (fs::exists(dirPath)) {
- for(fs::recursive_directory_iterator it(dirPath);
- it != fs::recursive_directory_iterator();
- ++it)
- {
- fileNames.insert(it->path().string());
- }
- for (const auto& f : fileNames) {
- auto data_ptr = ndn::io::load<T>(f, encoding);
- if (nullptr != data_ptr) {
- structures.push_back(*data_ptr);
- }
- }
- }
- structures.shrink_to_fit();
- return structures;
-}
-
-} // end anonymous
-
namespace ndn {
namespace ntorrent {
-// TODO(msweatt) Move this to a utility
-static vector<ndn::Data>
-packetize_file(const fs::path& filePath,
- const ndn::Name& commonPrefix,
- size_t dataPacketSize,
- size_t subManifestSize,
- size_t subManifestNum)
-{
- BOOST_ASSERT(0 < dataPacketSize);
- size_t APPROX_BUFFER_SIZE = std::numeric_limits<int>::max(); // 2 * 1024 * 1024 *1024
- auto file_size = fs::file_size(filePath);
- auto start_offset = subManifestNum * subManifestSize * dataPacketSize;
- // determine the number of bytes in this submanifest
- auto subManifestLength = subManifestSize * dataPacketSize;
- auto remainingFileLength = file_size - start_offset;
- subManifestLength = remainingFileLength < subManifestLength
- ? remainingFileLength
- : subManifestLength;
- vector<ndn::Data> packets;
- packets.reserve(subManifestLength/dataPacketSize + 1);
- fs::ifstream fs(filePath, fs::ifstream::binary);
- if (!fs) {
- BOOST_THROW_EXCEPTION(FileManifest::Error("IO Error when opening" + filePath.string()));
- }
- // ensure that buffer is large enough to contain whole packets
- // buffer size is either the entire file or the smallest number of data packets >= 2 GB
- auto buffer_size =
- subManifestLength < APPROX_BUFFER_SIZE ?
- subManifestLength :
- APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
- APPROX_BUFFER_SIZE :
- APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
- vector<char> file_bytes;
- file_bytes.reserve(buffer_size);
- size_t bytes_read = 0;
- fs.seekg(start_offset);
- while(fs && bytes_read < subManifestLength && !fs.eof()) {
- // read the file into the buffer
- fs.read(&file_bytes.front(), buffer_size);
- auto read_size = fs.gcount();
- if (fs.bad() || read_size < 0) {
- BOOST_THROW_EXCEPTION(FileManifest::Error("IO Error when reading" + filePath.string()));
- }
- bytes_read += read_size;
- char *curr_start = &file_bytes.front();
- for (size_t i = 0u; i < buffer_size; i += dataPacketSize) {
- // Build a packet from the data
- Name packetName = commonPrefix;
- packetName.appendSequenceNumber(packets.size());
- Data d(packetName);
- auto content_length = i + dataPacketSize > buffer_size ? buffer_size - i : dataPacketSize;
- d.setContent(encoding::makeBinaryBlock(tlv::Content, curr_start, content_length));
- curr_start += content_length;
- // append to the collection
- packets.push_back(d);
- }
- file_bytes.clear();
- // recompute the buffer_size
- buffer_size =
- subManifestLength - bytes_read < APPROX_BUFFER_SIZE ?
- subManifestLength - bytes_read :
- APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
- APPROX_BUFFER_SIZE :
- APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
- }
- fs.close();
- packets.shrink_to_fit();
- ndn::security::KeyChain key_chain;
- // sign all the packets
- for (auto& p : packets) {
- key_chain.sign(p, signingWithSha256());
- }
- return packets;
-}
-
-// =================================================================================================
-// Torrent Manager Utility Functions
-// ==================================================================================================
-
-static std::shared_ptr<Data>
-readDataPacket(const Name& packetFullName,
- const FileManifest& manifest,
- size_t subManifestSize,
- fs::fstream& is) {
- auto dataPacketSize = manifest.data_packet_size();
- auto start_offset = manifest.submanifest_number() * subManifestSize * dataPacketSize;
- auto packetNum = packetFullName.get(packetFullName.size() - 2).toSequenceNumber();
- // seek to packet
- is.sync();
- is.seekg(start_offset + packetNum * dataPacketSize);
- if (is.tellg() < 0) {
- std::cerr << "bad seek" << std::endl;
- }
- // read contents
- std::vector<char> bytes(dataPacketSize);
- is.read(&bytes.front(), dataPacketSize);
- auto read_size = is.gcount();
- if (is.bad() || read_size < 0) {
- std::cerr << "Bad read" << std::endl;
- return nullptr;
- }
- // construct packet
- auto packetName = packetFullName.getSubName(0, packetFullName.size() - 1);
- auto d = make_shared<Data>(packetName);
- d->setContent(encoding::makeBinaryBlock(tlv::Content, &bytes.front(), read_size));
- ndn::security::KeyChain key_chain;
- key_chain.sign(*d, signingWithSha256());
- return d->getFullName() == packetFullName ? d : nullptr;
-}
-
static vector<TorrentFile>
intializeTorrentSegments(const string& torrentFilePath, const Name& initialSegmentName)
{
security::KeyChain key_chain;
Name currSegmentFullName = initialSegmentName;
- vector<TorrentFile> torrentSegments = load_directory<TorrentFile>(torrentFilePath);
+ vector<TorrentFile> torrentSegments = IoUtil::load_directory<TorrentFile>(torrentFilePath);
+
// Starting with the initial segment name, verify the names, loading next name from torrentSegment
for (auto it = torrentSegments.begin(); it != torrentSegments.end(); ++it) {
TorrentFile& segment = *it;
@@ -191,10 +55,11 @@
{
security::KeyChain key_chain;
- vector<FileManifest> manifests = load_directory<FileManifest>(manifestPath);
+ vector<FileManifest> manifests = IoUtil::load_directory<FileManifest>(manifestPath);
if (manifests.empty()) {
return manifests;
}
+
// sign the manifests
std::for_each(manifests.begin(), manifests.end(),
[&key_chain](FileManifest& m){
@@ -249,11 +114,11 @@
vector<Data> packets;
auto subManifestNum = manifest.submanifest_number();
- packets = packetize_file(filePath,
- manifest.name(),
- manifest.data_packet_size(),
- subManifestSize,
- subManifestNum);
+ packets = IoUtil::packetize_file(filePath,
+ manifest.name(),
+ manifest.data_packet_size(),
+ subManifestSize,
+ subManifestNum);
auto catalog = manifest.catalog();
// Filter out invalid packet names
@@ -410,7 +275,7 @@
// Write the torrent file segment to disk...
if (writeTorrentSegment(file, path)) {
// if successfully written, seed this data
- seed(data);
+ seed(file);
}
const std::vector<Name>& manifestCatalog = file.getCatalog();
@@ -558,7 +423,6 @@
if (!fs::exists(filePath)) {
fs::create_directories(filePath.parent_path());
}
-
fileState = initializeFileState(m_dataPath,
*manifest_it,
m_subManifestSizes[manifest_it->file_name()]);
@@ -568,98 +432,62 @@
if (fileState.second[packetNum]) {
return false;
}
- auto dataPacketSize = manifest_it->data_packet_size();
- auto initial_offset = manifest_it->submanifest_number() * m_subManifestSizes[manifest_it->file_name()] * dataPacketSize;
- auto packetOffset = initial_offset + packetNum * dataPacketSize;
// write data to disk
- fileState.first->seekp(packetOffset);
- try {
- auto content = packet.getContent();
- std::vector<char> data(content.value_begin(), content.value_end());
- fileState.first->write(&data[0], data.size());
+ // TODO(msweatt) Fix this once code is merged
+ auto subManifestSize = m_subManifestSizes[manifest_it->file_name()];
+ if (IoUtil::writeData(packet, *manifest_it, subManifestSize, *fileState.first)) {
+ // update bitmap
+ fileState.second[packetNum] = true;
+ return true;
}
- catch (io::Error &e) {
- std::cerr << e.what() << std::endl;
- return false;
- }
- // update bitmap
- fileState.second[packetNum] = true;
- return true;
+ return false;
}
-bool TorrentManager::writeTorrentSegment(const TorrentFile& segment, const std::string& path)
+bool
+TorrentManager::writeTorrentSegment(const TorrentFile& segment, const std::string& path)
{
- // validate that this torrent segment belongs to our torrent
+ // validate the torrent
auto torrentPrefix = m_torrentFileName.getSubName(0, m_torrentFileName.size() - 1);
- if (!torrentPrefix.isPrefixOf(segment.getName())) {
- return false;
- }
-
- auto segmentNum = segment.getSegmentNumber();
// check if we already have it
- if (m_torrentSegments.end() != std::find(m_torrentSegments.begin(), m_torrentSegments.end(),
+ if (torrentPrefix.isPrefixOf(segment.getName()) &&
+ m_torrentSegments.end() == std::find(m_torrentSegments.begin(), m_torrentSegments.end(),
segment))
{
- return false;
- }
- // write to disk at path
- if (!fs::exists(path)) {
- fs::create_directories(path);
- }
- auto filename = path + to_string(segmentNum);
- // if there is already a file on disk for this torrent segment, determine if we should override
- if (fs::exists(filename)) {
- auto segmentOnDisk_ptr = io::load<TorrentFile>(filename);
- if (nullptr != segmentOnDisk_ptr && *segmentOnDisk_ptr == segment) {
- return false;
+ if(IoUtil::writeTorrentSegment(segment, path)) {
+ auto it = std::find_if(m_torrentSegments.begin(), m_torrentSegments.end(),
+ [&segment](const TorrentFile& t){
+ return segment.getSegmentNumber() < t.getSegmentNumber() ;
+ });
+ m_torrentSegments.insert(it, segment);
+ return true;
}
}
- io::save(segment, filename);
- // add to collection
- auto it = std::find_if(m_torrentSegments.begin(), m_torrentSegments.end(),
- [segmentNum](const TorrentFile& t){
- return t.getSegmentNumber() > segmentNum;
- });
- m_torrentSegments.insert(it, segment);
- return true;
+ return false;
}
+
bool TorrentManager::writeFileManifest(const FileManifest& manifest, const std::string& path)
{
- auto subManifestNum = manifest.submanifest_number();
- fs::path filename = path + manifest.file_name() + "/" + to_string(subManifestNum);
- // check if we already have it
- if (m_fileManifests.end() != std::find(m_fileManifests.begin(), m_fileManifests.end(),
- manifest))
+ if (m_fileManifests.end() == std::find(m_fileManifests.begin(), m_fileManifests.end(),
+ manifest))
{
- return false;
- }
-
- // write to disk at path
- if (!fs::exists(filename.parent_path())) {
- boost::filesystem::create_directories(filename.parent_path());
- }
- // if there is already a file on disk for this torrent segment, determine if we should override
- if (fs::exists(filename)) {
- auto submanifestOnDisk_ptr = io::load<FileManifest>(filename.string());
- if (nullptr != submanifestOnDisk_ptr && *submanifestOnDisk_ptr == manifest) {
- return false;
+ // update the state of the manager
+ if (0 == manifest.submanifest_number()) {
+ m_subManifestSizes[manifest.file_name()] = manifest.catalog().size();
+ }
+ if(IoUtil::writeFileManifest(manifest, path)) {
+ // add to collection
+ auto it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
+ [&manifest](const FileManifest& m){
+ return m.file_name() > manifest.file_name()
+ || (m.file_name() == manifest.file_name()
+ && (m.submanifest_number() > manifest.submanifest_number()));
+ });
+ m_fileManifests.insert(it, manifest);
+ return true;
}
}
- io::save(manifest, filename.string());
- // add to collection
- auto it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
- [&manifest](const FileManifest& m){
- return m.file_name() > manifest.file_name()
- || (m.file_name() == manifest.file_name()
- && (m.submanifest_number() > manifest.submanifest_number()));
- });
- // update the state of the manager
- m_fileManifests.insert(it, manifest);
- if (0 == manifest.submanifest_number()) {
- m_subManifestSizes[manifest.file_name()] = manifest.catalog().size();
- }
- return true;
+ return false;
}
void
@@ -680,9 +508,12 @@
FileManifest file(data.wireEncode());
// Write the file manifest segment to disk...
- if( writeFileManifest(file, path)) {
+ if(writeFileManifest(file, path)) {
seed(file);
}
+ else {
+ onFailed(interest.getName(), "Write Failed");
+ }
const std::vector<Name>& packetsCatalog = file.catalog();
packetNames->insert(packetNames->end(), packetsCatalog.begin(), packetsCatalog.end());
@@ -690,8 +521,9 @@
if (nextSegmentPtr != nullptr) {
this->downloadFileManifestSegment(*nextSegmentPtr, path, packetNames, onSuccess, onFailed);
}
- else
+ else {
onSuccess(*packetNames);
+ }
};
auto dataFailed = [packetNames, path, manifestName, onFailed, this]
@@ -752,10 +584,10 @@
auto filePath = m_dataPath + manifestFileName;
// TODO(msweatt) Explore why fileState stream does not work
fs::fstream is (filePath, fs::fstream::in | fs::fstream::binary);
- data = readDataPacket(interestName,
- *manifest_it,
- m_subManifestSizes[manifestFileName],
- is);
+ data = IoUtil::readDataPacket(interestName,
+ *manifest_it,
+ m_subManifestSizes[manifestFileName],
+ is);
}
}
}
diff --git a/src/util/io-util.cpp b/src/util/io-util.cpp
new file mode 100644
index 0000000..0d02475
--- /dev/null
+++ b/src/util/io-util.cpp
@@ -0,0 +1,210 @@
+#include "util/io-util.hpp"
+
+#include "file-manifest.hpp"
+#include "torrent-file.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include <ndn-cxx/security/key-chain.hpp>
+#include <ndn-cxx/security/signing-helpers.hpp>
+
+namespace fs = boost::filesystem;
+
+using std::string;
+using std::vector;
+
+namespace ndn {
+namespace ntorrent {
+
+std::vector<ndn::Data>
+IoUtil::packetize_file(const fs::path& filePath,
+ const ndn::Name& commonPrefix,
+ size_t dataPacketSize,
+ size_t subManifestSize,
+ size_t subManifestNum)
+{
+ BOOST_ASSERT(0 < dataPacketSize);
+ size_t APPROX_BUFFER_SIZE = std::numeric_limits<int>::max(); // 2 * 1024 * 1024 *1024
+ auto file_size = fs::file_size(filePath);
+ auto start_offset = subManifestNum * subManifestSize * dataPacketSize;
+ // determine the number of bytes in this submanifest
+ auto subManifestLength = subManifestSize * dataPacketSize;
+ auto remainingFileLength = file_size - start_offset;
+ subManifestLength = remainingFileLength < subManifestLength
+ ? remainingFileLength
+ : subManifestLength;
+ vector<ndn::Data> packets;
+ packets.reserve(subManifestLength/dataPacketSize + 1);
+ fs::ifstream fs(filePath, fs::ifstream::binary);
+ if (!fs) {
+ BOOST_THROW_EXCEPTION(Data::Error("IO Error when opening" + filePath.string()));
+ }
+ // ensure that buffer is large enough to contain whole packets
+ // buffer size is either the entire file or the smallest number of data packets >= 2 GB
+ auto buffer_size =
+ subManifestLength < APPROX_BUFFER_SIZE ?
+ subManifestLength :
+ APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
+ APPROX_BUFFER_SIZE :
+ APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
+ vector<char> file_bytes;
+ file_bytes.reserve(buffer_size);
+ size_t bytes_read = 0;
+ fs.seekg(start_offset);
+ while(fs && bytes_read < subManifestLength && !fs.eof()) {
+ // read the file into the buffer
+ fs.read(&file_bytes.front(), buffer_size);
+ auto read_size = fs.gcount();
+ if (fs.bad() || read_size < 0) {
+ BOOST_THROW_EXCEPTION(Data::Error("IO Error when reading" + filePath.string()));
+ }
+ bytes_read += read_size;
+ char *curr_start = &file_bytes.front();
+ for (size_t i = 0u; i < buffer_size; i += dataPacketSize) {
+ // Build a packet from the data
+ Name packetName = commonPrefix;
+ packetName.appendSequenceNumber(packets.size());
+ Data d(packetName);
+ auto content_length = i + dataPacketSize > buffer_size ? buffer_size - i : dataPacketSize;
+ d.setContent(encoding::makeBinaryBlock(tlv::Content, curr_start, content_length));
+ curr_start += content_length;
+ // append to the collection
+ packets.push_back(d);
+ }
+ file_bytes.clear();
+ // recompute the buffer_size
+ buffer_size =
+ subManifestLength - bytes_read < APPROX_BUFFER_SIZE ?
+ subManifestLength - bytes_read :
+ APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
+ APPROX_BUFFER_SIZE :
+ APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
+ }
+ fs.close();
+ packets.shrink_to_fit();
+ ndn::security::KeyChain key_chain;
+ // sign all the packets
+ for (auto& p : packets) {
+ key_chain.sign(p, signingWithSha256());
+ }
+ return packets;
+}
+
+bool IoUtil::writeTorrentSegment(const TorrentFile& segment, const std::string& path)
+{
+ // validate that this torrent segment belongs to our torrent
+ auto segmentNum = segment.getSegmentNumber();
+ // write to disk at path
+ if (!fs::exists(path)) {
+ fs::create_directories(path);
+ }
+ auto filename = path + to_string(segmentNum);
+ // if there is already a file on disk for this torrent segment, determine if we should override
+ if (fs::exists(filename)) {
+ auto segmentOnDisk_ptr = io::load<TorrentFile>(filename);
+ if (nullptr != segmentOnDisk_ptr && *segmentOnDisk_ptr == segment) {
+ return false;
+ }
+ }
+ io::save(segment, filename);
+ // add to collection
+ return true;
+}
+
+bool IoUtil::writeFileManifest(const FileManifest& manifest, const std::string& path)
+{
+ auto subManifestNum = manifest.submanifest_number();
+ fs::path filename = path + manifest.file_name() + "/" + to_string(subManifestNum);
+
+ // write to disk at path
+ if (!fs::exists(filename.parent_path())) {
+ boost::filesystem::create_directories(filename.parent_path());
+ }
+ // if there is already a file on disk for this file manifest, determine if we should override
+ if (fs::exists(filename)) {
+ auto submanifestOnDisk_ptr = io::load<FileManifest>(filename.string());
+ if (nullptr != submanifestOnDisk_ptr && *submanifestOnDisk_ptr == manifest) {
+ return false;
+ }
+ }
+ io::save(manifest, filename.string());
+ return true;
+}
+bool
+IoUtil::writeData(const Data& packet, const FileManifest& manifest, size_t subManifestSize, fs::fstream& os)
+{
+ auto packetName = packet.getName();
+ auto packetNum = packetName.get(packetName.size() - 1).toSequenceNumber();
+ auto dataPacketSize = manifest.data_packet_size();
+ auto initial_offset = manifest.submanifest_number() * subManifestSize * dataPacketSize;
+ auto packetOffset = initial_offset + packetNum * dataPacketSize;
+ // write data to disk
+ os.seekp(packetOffset);
+ try {
+ auto content = packet.getContent();
+ std::vector<char> data(content.value_begin(), content.value_end());
+ os.write(&data[0], data.size());
+ return true;
+ }
+ catch (io::Error &e) {
+ std::cerr << e.what() << std::endl;
+ return false;
+ }
+}
+
+std::shared_ptr<Data>
+IoUtil::readDataPacket(const Name& packetFullName,
+ const FileManifest& manifest,
+ size_t subManifestSize,
+ fs::fstream& is)
+{
+ auto dataPacketSize = manifest.data_packet_size();
+ auto start_offset = manifest.submanifest_number() * subManifestSize * dataPacketSize;
+ auto packetNum = packetFullName.get(packetFullName.size() - 2).toSequenceNumber();
+ // seek to packet
+ is.sync();
+ is.seekg(start_offset + packetNum * dataPacketSize);
+ if (is.tellg() < 0) {
+ std::cerr << "bad seek" << std::endl;
+ }
+ // read contents
+ std::vector<char> bytes(dataPacketSize);
+ is.read(&bytes.front(), dataPacketSize);
+ auto read_size = is.gcount();
+ if (is.bad() || read_size < 0) {
+ std::cerr << "Bad read" << std::endl;
+ return nullptr;
+ }
+ // construct packet
+ auto packetName = packetFullName.getSubName(0, packetFullName.size() - 1);
+ auto d = make_shared<Data>(packetName);
+ d->setContent(encoding::makeBinaryBlock(tlv::Content, &bytes.front(), read_size));
+ ndn::security::KeyChain key_chain;
+ key_chain.sign(*d, signingWithSha256());
+ return d->getFullName() == packetFullName ? d : nullptr;
+}
+
+IoUtil::NAME_TYPE
+IoUtil::findType(const Name& name)
+{
+ NAME_TYPE rval = UNKNOWN;
+ if (name.get(name.size() - 2).toUri() == "torrent-file" ||
+ name.get(name.size() - 3).toUri() == "torrent-file") {
+ rval = TORRENT_FILE;
+ }
+ else if (name.get(name.size() - 2).isSequenceNumber() &&
+ name.get(name.size() - 3).isSequenceNumber()) {
+ rval = DATA_PACKET;
+ }
+ else if (name.get(name.size() - 2).isSequenceNumber() &&
+ !(name.get(name.size() - 3).isSequenceNumber())) {
+ rval = FILE_MANIFEST;
+ }
+ return rval;
+}
+
+
+
+} // namespace ntorrent
+} // namespace ndn}
diff --git a/src/util/io-util.hpp b/src/util/io-util.hpp
new file mode 100644
index 0000000..d32ac65
--- /dev/null
+++ b/src/util/io-util.hpp
@@ -0,0 +1,156 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+* Copyright (c) 2016 Regents of the University of California.
+*
+* This file is part of the nTorrent codebase.
+*
+* nTorrent is free software: you can redistribute it and/or modify it under the
+* terms of the GNU Lesser General Public License as published by the Free Software
+* Foundation, either version 3 of the License, or (at your option) any later version.
+*
+* nTorrent is distributed in the hope that it will be useful, but WITHOUT ANY
+* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+* PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+*
+* You should have received copies of the GNU General Public License and GNU Lesser
+* General Public License along with nTorrent, e.g., in COPYING.md file. If not, see
+* <http://www.gnu.org/licenses/>.
+*
+* See AUTHORS for complete list of nTorrent authors and contributors.
+*/
+
+#ifndef INCLUDED_UTIL_IO_UTIL_H
+#define INCLUDED_UTIL_IO_UTIL_H
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include <ndn-cxx/data.hpp>
+#include <ndn-cxx/name.hpp>
+#include <ndn-cxx/util/io.hpp>
+
+#include <set>
+#include <string>
+#include <vector>
+
+namespace fs = boost::filesystem;
+
+namespace ndn {
+namespace ntorrent {
+
+class TorrentFile;
+class FileManifest;
+
+class IoUtil {
+ public:
+ /*
+ * An enum used to identify the type of content to which a name refers
+ */
+ enum NAME_TYPE
+ {
+ FILE_MANIFEST,
+ TORRENT_FILE,
+ DATA_PACKET,
+ UNKNOWN
+ };
+
+ template<typename T>
+ static std::vector<T>
+ load_directory(const std::string& dirPath,
+ ndn::io::IoEncoding encoding = ndn::io::IoEncoding::BASE_64);
+
+ static std::vector<ndn::Data>
+ packetize_file(const fs::path& filePath,
+ const ndn::Name& commonPrefix,
+ size_t dataPacketSize,
+ size_t subManifestSize,
+ size_t subManifestNum);
+
+ /*
+ * @brief Write the @p segment torrent segment to disk at the specified path.
+ * @param segment The torrent file segment to be written to disk
+ * @param path The path at which to write the torrent file segment
+ * Write the segment to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless @segment is a correct segment for the torrent file of
+ * this manager and @p path is the directory used for all segments of this torrent file.
+ */
+ static bool
+ writeTorrentSegment(const TorrentFile& segment, const std::string& path);
+
+ /*
+ * @brief Write the @p manifest file manifest to disk at the specified @p path.
+ * @param manifest The file manifest to be written to disk
+ * @param path The path at which to write the file manifest
+ * Write the file manifest to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless @manifest is a correct file manifest for a file in the
+ * torrent file of this manager and @p path is the directory used for all file manifests of this
+ * torrent file.
+ */
+ static bool
+ writeFileManifest(const FileManifest& manifest, const std::string& path);
+
+ /*
+ * @brief Write @p packet composed of torrent date to disk.
+ * @param packet The data packet to be written to the disk
+ * @param os The output stream to which to write
+ * Write the Data packet to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless the corresponding file manifest has already been
+ * downloaded.
+ */
+ static bool
+ writeData(const Data& packet,
+ const FileManifest& manifest,
+ size_t subManifestSize,
+ fs::fstream& os);
+
+ /*
+ * @brief Read a data packet from the provided stream
+ * @param packetFullName The fullname of the expected Data packet
+ * @param manifest The file manifest for the requested Data packet
+ * @param subManifestSize The number of Data packets in each catalog for this Data packet
+ * @param is The stream from which to read the packet
+ * Read the data packet from the @p is stream, validate it against the provided @p packetFullName
+ * and @p manifest, if successful return a pointer to the packet, otherwise return nullptr.
+ */
+ static std::shared_ptr<Data>
+ readDataPacket(const Name& packetFullName,
+ const FileManifest& manifest,
+ size_t subManifestSize,
+ fs::fstream& is);
+
+ /*
+ * @brief Return the type of the specified name
+ */
+ static IoUtil::NAME_TYPE
+ findType(const Name& name);
+
+};
+
+template<typename T>
+inline std::vector<T>
+IoUtil::load_directory(const std::string& dirPath,
+ ndn::io::IoEncoding encoding) {
+ std::vector<T> structures;
+ std::set<std::string> fileNames;
+ if (fs::exists(dirPath)) {
+ for(fs::recursive_directory_iterator it(dirPath);
+ it != fs::recursive_directory_iterator();
+ ++it)
+ {
+ fileNames.insert(it->path().string());
+ }
+ for (const auto& f : fileNames) {
+ auto data_ptr = ndn::io::load<T>(f, encoding);
+ if (nullptr != data_ptr) {
+ structures.push_back(*data_ptr);
+ }
+ }
+ }
+ structures.shrink_to_fit();
+ return structures;
+}
+
+} // namespace ntorrent
+} // namespace ndn
+
+#endif // INCLUDED_UTIL_IO_UTIL_H
\ No newline at end of file