Abstracted out some repeated I/O helpers into a IoUtil and updated affected components.
Change-Id: I151b880b29c526cb1ccb9717fabc63d1842cd1c1
diff --git a/src/util/io-util.cpp b/src/util/io-util.cpp
new file mode 100644
index 0000000..0d02475
--- /dev/null
+++ b/src/util/io-util.cpp
@@ -0,0 +1,210 @@
+#include "util/io-util.hpp"
+
+#include "file-manifest.hpp"
+#include "torrent-file.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include <ndn-cxx/security/key-chain.hpp>
+#include <ndn-cxx/security/signing-helpers.hpp>
+
+namespace fs = boost::filesystem;
+
+using std::string;
+using std::vector;
+
+namespace ndn {
+namespace ntorrent {
+
+std::vector<ndn::Data>
+IoUtil::packetize_file(const fs::path& filePath,
+ const ndn::Name& commonPrefix,
+ size_t dataPacketSize,
+ size_t subManifestSize,
+ size_t subManifestNum)
+{
+ BOOST_ASSERT(0 < dataPacketSize);
+ size_t APPROX_BUFFER_SIZE = std::numeric_limits<int>::max(); // 2 * 1024 * 1024 *1024
+ auto file_size = fs::file_size(filePath);
+ auto start_offset = subManifestNum * subManifestSize * dataPacketSize;
+ // determine the number of bytes in this submanifest
+ auto subManifestLength = subManifestSize * dataPacketSize;
+ auto remainingFileLength = file_size - start_offset;
+ subManifestLength = remainingFileLength < subManifestLength
+ ? remainingFileLength
+ : subManifestLength;
+ vector<ndn::Data> packets;
+ packets.reserve(subManifestLength/dataPacketSize + 1);
+ fs::ifstream fs(filePath, fs::ifstream::binary);
+ if (!fs) {
+ BOOST_THROW_EXCEPTION(Data::Error("IO Error when opening" + filePath.string()));
+ }
+ // ensure that buffer is large enough to contain whole packets
+ // buffer size is either the entire file or the smallest number of data packets >= 2 GB
+ auto buffer_size =
+ subManifestLength < APPROX_BUFFER_SIZE ?
+ subManifestLength :
+ APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
+ APPROX_BUFFER_SIZE :
+ APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
+ vector<char> file_bytes;
+ file_bytes.reserve(buffer_size);
+ size_t bytes_read = 0;
+ fs.seekg(start_offset);
+ while(fs && bytes_read < subManifestLength && !fs.eof()) {
+ // read the file into the buffer
+ fs.read(&file_bytes.front(), buffer_size);
+ auto read_size = fs.gcount();
+ if (fs.bad() || read_size < 0) {
+ BOOST_THROW_EXCEPTION(Data::Error("IO Error when reading" + filePath.string()));
+ }
+ bytes_read += read_size;
+ char *curr_start = &file_bytes.front();
+ for (size_t i = 0u; i < buffer_size; i += dataPacketSize) {
+ // Build a packet from the data
+ Name packetName = commonPrefix;
+ packetName.appendSequenceNumber(packets.size());
+ Data d(packetName);
+ auto content_length = i + dataPacketSize > buffer_size ? buffer_size - i : dataPacketSize;
+ d.setContent(encoding::makeBinaryBlock(tlv::Content, curr_start, content_length));
+ curr_start += content_length;
+ // append to the collection
+ packets.push_back(d);
+ }
+ file_bytes.clear();
+ // recompute the buffer_size
+ buffer_size =
+ subManifestLength - bytes_read < APPROX_BUFFER_SIZE ?
+ subManifestLength - bytes_read :
+ APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
+ APPROX_BUFFER_SIZE :
+ APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
+ }
+ fs.close();
+ packets.shrink_to_fit();
+ ndn::security::KeyChain key_chain;
+ // sign all the packets
+ for (auto& p : packets) {
+ key_chain.sign(p, signingWithSha256());
+ }
+ return packets;
+}
+
+bool IoUtil::writeTorrentSegment(const TorrentFile& segment, const std::string& path)
+{
+ // validate that this torrent segment belongs to our torrent
+ auto segmentNum = segment.getSegmentNumber();
+ // write to disk at path
+ if (!fs::exists(path)) {
+ fs::create_directories(path);
+ }
+ auto filename = path + to_string(segmentNum);
+ // if there is already a file on disk for this torrent segment, determine if we should override
+ if (fs::exists(filename)) {
+ auto segmentOnDisk_ptr = io::load<TorrentFile>(filename);
+ if (nullptr != segmentOnDisk_ptr && *segmentOnDisk_ptr == segment) {
+ return false;
+ }
+ }
+ io::save(segment, filename);
+ // add to collection
+ return true;
+}
+
+bool IoUtil::writeFileManifest(const FileManifest& manifest, const std::string& path)
+{
+ auto subManifestNum = manifest.submanifest_number();
+ fs::path filename = path + manifest.file_name() + "/" + to_string(subManifestNum);
+
+ // write to disk at path
+ if (!fs::exists(filename.parent_path())) {
+ boost::filesystem::create_directories(filename.parent_path());
+ }
+ // if there is already a file on disk for this file manifest, determine if we should override
+ if (fs::exists(filename)) {
+ auto submanifestOnDisk_ptr = io::load<FileManifest>(filename.string());
+ if (nullptr != submanifestOnDisk_ptr && *submanifestOnDisk_ptr == manifest) {
+ return false;
+ }
+ }
+ io::save(manifest, filename.string());
+ return true;
+}
+bool
+IoUtil::writeData(const Data& packet, const FileManifest& manifest, size_t subManifestSize, fs::fstream& os)
+{
+ auto packetName = packet.getName();
+ auto packetNum = packetName.get(packetName.size() - 1).toSequenceNumber();
+ auto dataPacketSize = manifest.data_packet_size();
+ auto initial_offset = manifest.submanifest_number() * subManifestSize * dataPacketSize;
+ auto packetOffset = initial_offset + packetNum * dataPacketSize;
+ // write data to disk
+ os.seekp(packetOffset);
+ try {
+ auto content = packet.getContent();
+ std::vector<char> data(content.value_begin(), content.value_end());
+ os.write(&data[0], data.size());
+ return true;
+ }
+ catch (io::Error &e) {
+ std::cerr << e.what() << std::endl;
+ return false;
+ }
+}
+
+std::shared_ptr<Data>
+IoUtil::readDataPacket(const Name& packetFullName,
+ const FileManifest& manifest,
+ size_t subManifestSize,
+ fs::fstream& is)
+{
+ auto dataPacketSize = manifest.data_packet_size();
+ auto start_offset = manifest.submanifest_number() * subManifestSize * dataPacketSize;
+ auto packetNum = packetFullName.get(packetFullName.size() - 2).toSequenceNumber();
+ // seek to packet
+ is.sync();
+ is.seekg(start_offset + packetNum * dataPacketSize);
+ if (is.tellg() < 0) {
+ std::cerr << "bad seek" << std::endl;
+ }
+ // read contents
+ std::vector<char> bytes(dataPacketSize);
+ is.read(&bytes.front(), dataPacketSize);
+ auto read_size = is.gcount();
+ if (is.bad() || read_size < 0) {
+ std::cerr << "Bad read" << std::endl;
+ return nullptr;
+ }
+ // construct packet
+ auto packetName = packetFullName.getSubName(0, packetFullName.size() - 1);
+ auto d = make_shared<Data>(packetName);
+ d->setContent(encoding::makeBinaryBlock(tlv::Content, &bytes.front(), read_size));
+ ndn::security::KeyChain key_chain;
+ key_chain.sign(*d, signingWithSha256());
+ return d->getFullName() == packetFullName ? d : nullptr;
+}
+
+IoUtil::NAME_TYPE
+IoUtil::findType(const Name& name)
+{
+ NAME_TYPE rval = UNKNOWN;
+ if (name.get(name.size() - 2).toUri() == "torrent-file" ||
+ name.get(name.size() - 3).toUri() == "torrent-file") {
+ rval = TORRENT_FILE;
+ }
+ else if (name.get(name.size() - 2).isSequenceNumber() &&
+ name.get(name.size() - 3).isSequenceNumber()) {
+ rval = DATA_PACKET;
+ }
+ else if (name.get(name.size() - 2).isSequenceNumber() &&
+ !(name.get(name.size() - 3).isSequenceNumber())) {
+ rval = FILE_MANIFEST;
+ }
+ return rval;
+}
+
+
+
+} // namespace ntorrent
+} // namespace ndn}
diff --git a/src/util/io-util.hpp b/src/util/io-util.hpp
new file mode 100644
index 0000000..d32ac65
--- /dev/null
+++ b/src/util/io-util.hpp
@@ -0,0 +1,156 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+* Copyright (c) 2016 Regents of the University of California.
+*
+* This file is part of the nTorrent codebase.
+*
+* nTorrent is free software: you can redistribute it and/or modify it under the
+* terms of the GNU Lesser General Public License as published by the Free Software
+* Foundation, either version 3 of the License, or (at your option) any later version.
+*
+* nTorrent is distributed in the hope that it will be useful, but WITHOUT ANY
+* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+* PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+*
+* You should have received copies of the GNU General Public License and GNU Lesser
+* General Public License along with nTorrent, e.g., in COPYING.md file. If not, see
+* <http://www.gnu.org/licenses/>.
+*
+* See AUTHORS for complete list of nTorrent authors and contributors.
+*/
+
+#ifndef INCLUDED_UTIL_IO_UTIL_H
+#define INCLUDED_UTIL_IO_UTIL_H
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include <ndn-cxx/data.hpp>
+#include <ndn-cxx/name.hpp>
+#include <ndn-cxx/util/io.hpp>
+
+#include <set>
+#include <string>
+#include <vector>
+
+namespace fs = boost::filesystem;
+
+namespace ndn {
+namespace ntorrent {
+
+class TorrentFile;
+class FileManifest;
+
+class IoUtil {
+ public:
+ /*
+ * An enum used to identify the type of content to which a name refers
+ */
+ enum NAME_TYPE
+ {
+ FILE_MANIFEST,
+ TORRENT_FILE,
+ DATA_PACKET,
+ UNKNOWN
+ };
+
+ template<typename T>
+ static std::vector<T>
+ load_directory(const std::string& dirPath,
+ ndn::io::IoEncoding encoding = ndn::io::IoEncoding::BASE_64);
+
+ static std::vector<ndn::Data>
+ packetize_file(const fs::path& filePath,
+ const ndn::Name& commonPrefix,
+ size_t dataPacketSize,
+ size_t subManifestSize,
+ size_t subManifestNum);
+
+ /*
+ * @brief Write the @p segment torrent segment to disk at the specified path.
+ * @param segment The torrent file segment to be written to disk
+ * @param path The path at which to write the torrent file segment
+ * Write the segment to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless @segment is a correct segment for the torrent file of
+ * this manager and @p path is the directory used for all segments of this torrent file.
+ */
+ static bool
+ writeTorrentSegment(const TorrentFile& segment, const std::string& path);
+
+ /*
+ * @brief Write the @p manifest file manifest to disk at the specified @p path.
+ * @param manifest The file manifest to be written to disk
+ * @param path The path at which to write the file manifest
+ * Write the file manifest to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless @manifest is a correct file manifest for a file in the
+ * torrent file of this manager and @p path is the directory used for all file manifests of this
+ * torrent file.
+ */
+ static bool
+ writeFileManifest(const FileManifest& manifest, const std::string& path);
+
+ /*
+ * @brief Write @p packet composed of torrent date to disk.
+ * @param packet The data packet to be written to the disk
+ * @param os The output stream to which to write
+ * Write the Data packet to disk, return 'true' if data successfully written to disk 'false'
+ * otherwise. Behavior is undefined unless the corresponding file manifest has already been
+ * downloaded.
+ */
+ static bool
+ writeData(const Data& packet,
+ const FileManifest& manifest,
+ size_t subManifestSize,
+ fs::fstream& os);
+
+ /*
+ * @brief Read a data packet from the provided stream
+ * @param packetFullName The fullname of the expected Data packet
+ * @param manifest The file manifest for the requested Data packet
+ * @param subManifestSize The number of Data packets in each catalog for this Data packet
+ * @param is The stream from which to read the packet
+ * Read the data packet from the @p is stream, validate it against the provided @p packetFullName
+ * and @p manifest, if successful return a pointer to the packet, otherwise return nullptr.
+ */
+ static std::shared_ptr<Data>
+ readDataPacket(const Name& packetFullName,
+ const FileManifest& manifest,
+ size_t subManifestSize,
+ fs::fstream& is);
+
+ /*
+ * @brief Return the type of the specified name
+ */
+ static IoUtil::NAME_TYPE
+ findType(const Name& name);
+
+};
+
+template<typename T>
+inline std::vector<T>
+IoUtil::load_directory(const std::string& dirPath,
+ ndn::io::IoEncoding encoding) {
+ std::vector<T> structures;
+ std::set<std::string> fileNames;
+ if (fs::exists(dirPath)) {
+ for(fs::recursive_directory_iterator it(dirPath);
+ it != fs::recursive_directory_iterator();
+ ++it)
+ {
+ fileNames.insert(it->path().string());
+ }
+ for (const auto& f : fileNames) {
+ auto data_ptr = ndn::io::load<T>(f, encoding);
+ if (nullptr != data_ptr) {
+ structures.push_back(*data_ptr);
+ }
+ }
+ }
+ structures.shrink_to_fit();
+ return structures;
+}
+
+} // namespace ntorrent
+} // namespace ndn
+
+#endif // INCLUDED_UTIL_IO_UTIL_H
\ No newline at end of file