Create utility method to generate file manifests
Refs: #3431
Change-Id: I1a0f06b87689e71349b4fa9346ebc29e785d86cc
diff --git a/src/file-manifest.cpp b/src/file-manifest.cpp
index fe4fec8..71ba3a6 100644
--- a/src/file-manifest.cpp
+++ b/src/file-manifest.cpp
@@ -20,11 +20,23 @@
*/
#include "file-manifest.hpp"
+#include <limits>
+
#include <boost/assert.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/lexical_cast.hpp>
#include <boost/range/adaptors.hpp>
+#include <boost/range/irange.hpp>
#include <boost/throw_exception.hpp>
+#include <ndn-cxx/security/key-chain.hpp>
+#include <ndn-cxx/security/signing-helpers.hpp>
using std::vector;
+using std::streamsize;
+using boost::irange;
+
+namespace fs = boost::filesystem;
namespace ndn {
namespace ntorrent {
@@ -35,6 +47,168 @@
static_assert(std::is_base_of<Data::Error, FileManifest::Error>::value,
"FileManifest::Error should inherit from Data::Error");
+static ndn::Name
+get_name_of_manifest(const std::string& filePath, const Name& manifestPrefix)
+{
+ Name full_path(fs::system_complete(filePath).string());
+ // Search the filePath for the leading component that matches
+ auto name_component_iter = std::find(full_path.rbegin(),
+ full_path.rend(),
+ *manifestPrefix.rbegin());
+
+ if (full_path.rend() == name_component_iter) {
+ BOOST_THROW_EXCEPTION(FileManifest::Error("No matching name component between" +
+ manifestPrefix.toUri() + " and " +
+ full_path.toUri()));
+ }
+ ndn::Name manifestName;
+ // Rebuild the name to be the suffix from the matching component
+ for (auto it = (name_component_iter.base() - 1); full_path.end() != it; ++it) {
+ manifestName.append(*it);
+ }
+ return manifestName;
+}
+
+static std::vector<Data>
+packetize_file(const fs::path& filePath,
+ const ndn::Name& commonPrefix,
+ size_t dataPacketSize,
+ size_t subManifestSize,
+ size_t subManifestNum)
+{
+ BOOST_ASSERT(0 < dataPacketSize);
+ size_t APPROX_BUFFER_SIZE = std::numeric_limits<int>::max(); // 2 * 1024 * 1024 *1024
+ auto file_size = fs::file_size(filePath);
+ auto start_offset = subManifestNum * subManifestSize * dataPacketSize;
+ // determine the number of bytes in this submanifest
+ auto subManifestLength = subManifestSize * dataPacketSize;
+ auto remainingFileLength = file_size - start_offset;
+ subManifestLength = remainingFileLength < subManifestLength
+ ? remainingFileLength
+ : subManifestLength;
+ std::vector<Data> packets;
+ packets.reserve(subManifestLength/dataPacketSize + 1);
+ fs::ifstream fs(filePath, fs::ifstream::binary);
+ if (!fs) {
+ BOOST_THROW_EXCEPTION(FileManifest::Error("IO Error when opening" + filePath.string()));
+ }
+ // ensure that buffer is large enough to contain whole packets
+ // buffer size is either the entire file or the smallest number of data packets >= 2 GB
+ auto buffer_size =
+ subManifestLength < APPROX_BUFFER_SIZE ?
+ subManifestLength :
+ APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
+ APPROX_BUFFER_SIZE :
+ APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
+ std::vector<char> file_bytes;
+ file_bytes.reserve(buffer_size);
+ size_t bytes_read = 0;
+ fs.seekg(start_offset);
+ while(fs && bytes_read < subManifestLength && !fs.eof()) {
+ // read the file into the buffer
+ fs.read(&file_bytes.front(), buffer_size);
+ auto read_size = fs.gcount();
+ if (fs.bad() || read_size < 0) {
+
+ BOOST_THROW_EXCEPTION(FileManifest::Error("IO Error when reading" + filePath.string()));
+ }
+ bytes_read += read_size;
+ char *curr_start = &file_bytes.front();
+ for (size_t i = 0u; i < buffer_size; i += dataPacketSize) {
+ // Build a packet from the data
+ Name packetName = commonPrefix;
+ packetName.appendSequenceNumber(packets.size());
+ Data d(packetName);
+ auto content_length = i + dataPacketSize > buffer_size ? buffer_size - i : dataPacketSize;
+ d.setContent(encoding::makeBinaryBlock(tlv::Content, curr_start, content_length));
+ curr_start += content_length;
+ // append to the collection
+ packets.push_back(d);
+ }
+ file_bytes.clear();
+ // recompute the buffer_size
+ buffer_size =
+ subManifestLength - bytes_read < APPROX_BUFFER_SIZE ?
+ subManifestLength - bytes_read :
+ APPROX_BUFFER_SIZE % dataPacketSize == 0 ?
+ APPROX_BUFFER_SIZE :
+ APPROX_BUFFER_SIZE + dataPacketSize - (APPROX_BUFFER_SIZE % dataPacketSize);
+ }
+ fs.close();
+ packets.shrink_to_fit();
+ security::KeyChain key_chain;
+ // sign all the packets
+ for (auto& p : packets) {
+ key_chain.sign(p, signingWithSha256());
+ }
+ return packets;
+}
+
+// CLASS METHODS
+std::pair<std::vector<FileManifest>, std::vector<Data>>
+FileManifest::generate(const std::string& filePath,
+ const Name& manifestPrefix,
+ size_t subManifestSize,
+ size_t dataPacketSize,
+ bool returnData)
+{
+ BOOST_ASSERT(0 < subManifestSize);
+ BOOST_ASSERT(0 < dataPacketSize);
+ std::vector<FileManifest> manifests;
+ fs::path path(filePath);
+ if (!fs::exists(path)) {
+ BOOST_THROW_EXCEPTION(Error(filePath + ": no such file."));
+ }
+ size_t file_length = fs::file_size(filePath);
+ // If the file_length is not evenly divisible by subManifestSize add 1, otherwise 0
+ size_t numSubManifests = file_length / (subManifestSize * dataPacketSize) +
+ !!(file_length % (subManifestSize * dataPacketSize));
+ // Find the prefix for the Catalog
+ auto manifestName = get_name_of_manifest(filePath, manifestPrefix);
+ std::vector<Data> allPackets;
+ if (returnData) {
+ allPackets.reserve(numSubManifests * subManifestSize);
+ }
+ manifests.reserve(numSubManifests);
+ for (auto subManifestNum : irange<size_t>(0, numSubManifests)) {
+ auto curr_manifest_name = manifestName;
+ curr_manifest_name.append("manifest" + boost::lexical_cast<std::string>(subManifestNum));
+ // append the packet number
+ curr_manifest_name.appendSequenceNumber(manifests.size());
+ auto subManifestPrefix = manifestPrefix;
+ subManifestPrefix.appendNumber(subManifestNum);
+ FileManifest curr_manifest(curr_manifest_name, dataPacketSize, manifestPrefix);
+ auto packets = packetize_file(path,
+ subManifestPrefix,
+ dataPacketSize,
+ subManifestSize,
+ subManifestNum);
+ if (returnData) {
+ allPackets.insert(allPackets.end(), packets.begin(), packets.end());
+ }
+ curr_manifest.reserve(packets.size());
+ // Collect all the Data packets into the sub-manifests
+ for (const auto& p: packets) {
+ curr_manifest.push_back(p.getFullName());
+ }
+ // append the last manifest
+ manifests.push_back(curr_manifest);
+ }
+ allPackets.shrink_to_fit();
+ manifests.shrink_to_fit();
+ // Set all the submanifest_ptrs and sign all the manifests
+ security::KeyChain key_chain;
+ manifests.back().finalize();
+ key_chain.sign(manifests.back(), signingWithSha256());
+ for (auto it = manifests.rbegin() + 1; it != manifests.rend(); ++it) {
+ auto next = it - 1;
+ it->set_submanifest_ptr(std::make_shared<Name>(next->getFullName()));
+ it->finalize();
+ key_chain.sign(*it, signingWithSha256());
+ }
+ return {manifests, allPackets};
+}
+
void
FileManifest::wireDecode(const Block& wire)
{
@@ -72,7 +246,7 @@
suffixCatalog.reserve(m_catalog.size());
for (auto name: m_catalog) {
if (!m_catalogPrefix.isPrefixOf(name)) {
- BOOST_THROW_EXCEPTION(Error(name.toUri() + "does not have the prefix"
+ BOOST_THROW_EXCEPTION(Error(name.toUri() + " does not have the prefix "
+ m_catalogPrefix.toUri()));
}
name = name.getSubName(m_catalogPrefix.size());
@@ -106,12 +280,13 @@
{
BOOST_ASSERT(name != m_catalogPrefix);
BOOST_ASSERT(m_catalogPrefix.isPrefixOf(name));
- m_catalog.push_back(name);
+ // TODO(msweatt) Change this to use the copy constructor once bug fixed in Name()
+ m_catalog.push_back(name.toUri());
}
bool
FileManifest::remove(const ndn::Name& name) {
- auto it = std::find(m_catalog.begin(), m_catalog.end(), name);
+ const auto it = std::find(m_catalog.begin(), m_catalog.end(), name);
if (m_catalog.end() == it) {
return false;
}
diff --git a/src/file-manifest.hpp b/src/file-manifest.hpp
index 6ce957e..b3cf3ff 100644
--- a/src/file-manifest.hpp
+++ b/src/file-manifest.hpp
@@ -22,9 +22,9 @@
#define INCLUDED_FILE_MANIFEST_HPP
#include <cstring>
-#include <list>
#include <memory>
#include <string>
+#include <utility>
#include <vector>
#include <ndn-cxx/data.hpp>
@@ -53,6 +53,39 @@
};
public:
+ // CLASS METHODS
+ static std::vector<FileManifest>
+ generate(const std::string& filePath,
+ const ndn::Name& manifestPrefix,
+ size_t subManifestSize,
+ size_t dataPacketSize);
+
+
+ static std::pair<std::vector<FileManifest>, std::vector<Data>>
+ generate(const std::string& filePath,
+ const ndn::Name& manifestPrefix,
+ size_t subManifestSize,
+ size_t dataPacketSize,
+ bool returnData);
+ /**
+ * \brief Generates the FileManifest(s) and Data packets for the file at the specified 'filePath'
+ *
+ * @param filePath The path to the file for which we are to create a manifest
+ * @param manifestPrefix The prefix to be used for the name of this manifest
+ * @param subManifestSize The maximum number of data packets to be included in a sub-manifest
+ * @param dataPacketSize The maximum number of bytes per Data packet packets for the file
+ * @param returnData If true also return the Data
+ *
+ * @throws Error if there is any I/O issue when trying to read the filePath.
+ *
+ * Generates the FileManfiest(s) for the file at the specified 'filePath', splitting the manifest
+ * into sub-manifests of size at most the specified 'subManifestSize'. Each sub-manifest is
+ * composed of a catalog of Data packets of at most the specified 'dataPacketSize'. Returns all
+ * of the manifests that were created in order. The behavior is undefined unless the
+ * trailing component of of the manifestPrefix is a subComponent filePath and
+ '* O < subManifestSize' and '0 < dataPacketSize'.
+ */
+
// CREATORS
FileManifest() = delete;
@@ -129,9 +162,17 @@
/// Assigns the value of the specific 'rhs' object to this object.
void
+ set_submanifest_ptr(std::shared_ptr<Name> subManifestPtr);
+ /// Sets the sub-manifest pointer of manifest to the specified 'subManifestPtr'
+
+ void
push_back(const Name& name);
/// Appends a Name to the catalog
+ void
+ reserve(size_t capacity);
+ /// Reserve memory in the catalog adequate to hold at least 'capacity' Names.
+
bool
remove(const Name& name);
/// If 'name' in catalog, removes first instance and returns 'true', otherwise returns 'false'.
@@ -179,6 +220,16 @@
/// Returns 'true' if 'lhs' and 'rhs' have different values, and 'false' otherwise.
inline
+std::vector<FileManifest>
+FileManifest::generate(const std::string& filePath,
+ const ndn::Name& manifestPrefix,
+ size_t subManifestSize,
+ size_t dataPacketSize)
+{
+ return generate(filePath, manifestPrefix, subManifestSize, dataPacketSize, false).first;
+}
+
+inline
FileManifest::FileManifest(
const Name& name,
size_t dataPacketSize,
@@ -193,6 +244,7 @@
{
}
+
inline
FileManifest::FileManifest(
const Name& name,
@@ -249,6 +301,18 @@
return m_submanifestPtr;
}
+inline void
+FileManifest::set_submanifest_ptr(std::shared_ptr<Name> subManifestPtr)
+{
+ m_submanifestPtr = subManifestPtr;
+}
+
+inline void
+FileManifest::reserve(size_t capacity)
+{
+ m_catalog.reserve(capacity);
+}
+
} // end ntorrent
} // end ndn