Enable torrent manager to check whether we already have a requested torrent file segment, manifest file segment, data packet before downloading
Refs: #3505
Change-Id: I97472459a774b1a858462f82a5b7d9606bd0843a
diff --git a/src/torrent-manager.cpp b/src/torrent-manager.cpp
index c06db40..4a0eaff 100644
--- a/src/torrent-manager.cpp
+++ b/src/torrent-manager.cpp
@@ -436,18 +436,44 @@
void
TorrentManager::downloadTorrentFile(const std::string& path,
- DataReceivedCallback onSuccess,
+ TorrentFileReceivedCallback onSuccess,
FailedCallback onFailed)
{
+ shared_ptr<Name> searchRes = this->findTorrentFileSegmentToDownload();
auto manifestNames = make_shared<std::vector<Name>>();
- this->downloadTorrentFileSegment(m_torrentFileName, path, manifestNames,
+ if (searchRes == nullptr) {
+ this->findFileManifestsToDownload(*manifestNames);
+ if (manifestNames->empty()) {
+ auto packetNames = make_shared<std::vector<Name>>();
+ this->findAllMissingDataPackets(*packetNames);
+ onSuccess(*packetNames);
+ return;
+ }
+ else {
+ onSuccess(*manifestNames);
+ return;
+ }
+ }
+ this->downloadTorrentFileSegment(*searchRes, path, manifestNames,
true, onSuccess, onFailed);
}
std::vector<Name>
TorrentManager::downloadTorrentFile(const std::string& path)
{
+ shared_ptr<Name> searchRes = this->findTorrentFileSegmentToDownload();
auto manifestNames = make_shared<std::vector<Name>>();
+ if (searchRes == nullptr) {
+ this->findFileManifestsToDownload(*manifestNames);
+ if (manifestNames->empty()) {
+ auto packetNames = make_shared<std::vector<Name>>();
+ this->findAllMissingDataPackets(*packetNames);
+ return *packetNames;
+ }
+ else {
+ return *manifestNames;
+ }
+ }
this->downloadTorrentFileSegment(m_torrentFileName, path, manifestNames,
false, {}, {});
return *manifestNames;
@@ -458,7 +484,7 @@
const std::string& path,
std::shared_ptr<std::vector<Name>> manifestNames,
bool async,
- DataReceivedCallback onSuccess,
+ TorrentFileReceivedCallback onSuccess,
FailedCallback onFailed)
{
shared_ptr<Interest> interest = createInterest(name);
@@ -469,6 +495,10 @@
m_stats_table_iter->incrementReceivedData();
m_retries = 0;
+ if (async) {
+ manifestNames->clear();
+ }
+
TorrentFile file(data.wireEncode());
// Write the torrent file segment to disk...
@@ -480,7 +510,7 @@
shared_ptr<Name> nextSegmentPtr = file.getTorrentFilePtr();
if (async) {
- onSuccess(file.getName());
+ onSuccess(*manifestNames);
}
if (nextSegmentPtr != nullptr) {
this->downloadTorrentFileSegment(*nextSegmentPtr, path, manifestNames,
@@ -517,8 +547,14 @@
TorrentManager::ManifestReceivedCallback onSuccess,
TorrentManager::FailedCallback onFailed)
{
+ shared_ptr<Name> searchRes = findManifestSegmentToDownload(manifestName);
auto packetNames = make_shared<std::vector<Name>>();
- this->downloadFileManifestSegment(manifestName, path, packetNames, onSuccess, onFailed);
+ if (searchRes == nullptr) {
+ this->findDataPacketsToDownload(manifestName, *packetNames);
+ onSuccess(*packetNames);
+ return;
+ }
+ this->downloadFileManifestSegment(*searchRes, path, packetNames, onSuccess, onFailed);
}
void
@@ -570,6 +606,11 @@
DataReceivedCallback onSuccess,
FailedCallback onFailed)
{
+ if (this->dataAlreadyDownloaded(packetName)) {
+ onSuccess(packetName);
+ return;
+ }
+
shared_ptr<Interest> interest = this->createInterest(packetName);
auto dataReceived = [onSuccess, onFailed, this]
@@ -624,5 +665,121 @@
return interest;
}
+shared_ptr<Name>
+TorrentManager::findTorrentFileSegmentToDownload()
+{
+ // if we have no segments
+ if (m_torrentSegments.empty()) {
+ return make_shared<Name>(m_torrentFileName);
+ }
+ // otherwise just return the next segment ptr of the last segment we have
+ return m_torrentSegments.back().getTorrentFilePtr();
+}
+
+shared_ptr<Name>
+TorrentManager::findManifestSegmentToDownload(const Name& manifestName)
+{
+ //sequentially find whether we have downloaded any segments of this manifest file
+ Name manifestPrefix = manifestName.getSubName(0, manifestName.size() - 2);
+ auto it = std::find_if(m_fileManifests.rbegin(), m_fileManifests.rend(),
+ [&manifestPrefix] (const FileManifest& f) {
+ return manifestPrefix.isPrefixOf(f.getName());
+ });
+
+ // if we do not have any segments of the file manifest
+ if (it == m_fileManifests.rend()) {
+ return make_shared<Name>(manifestName);
+ }
+
+ // if we already have the requested segment of the file manifest
+ if (it->submanifest_number() >= manifestName.get(manifestName.size() - 2).toSequenceNumber()) {
+ return it->submanifest_ptr();
+ }
+ // if we do not have the requested segment
+ else {
+ return make_shared<Name>(manifestName);
+ }
+}
+
+bool
+TorrentManager::dataAlreadyDownloaded(const Name& dataName)
+{
+
+ auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
+ [&dataName](const FileManifest& m) {
+ return m.getName().isPrefixOf(dataName);
+ });
+
+ // if we do not have the file manifest, just return false
+ if (manifest_it == m_fileManifests.end()) {
+ return false;
+ }
+
+ // find the pair of (std::shared_ptr<fs::fstream>, std::vector<bool>)
+ // that corresponds to the specific submanifest
+ auto& fileState = m_fileStates[manifest_it->getFullName()];
+
+ auto dataNum = dataName.get(dataName.size() - 2).toSequenceNumber();
+
+ // find whether we have the requested packet from the bitmap
+ return fileState.second[dataNum];
+}
+
+void
+TorrentManager::findFileManifestsToDownload(std::vector<Name>& manifestNames)
+{
+ std::vector<Name> manifests;
+ // insert the first segment name of all the file manifests to the vector
+ for (auto i = m_torrentSegments.begin(); i != m_torrentSegments.end(); i++) {
+ manifests.insert(manifests.end(), i->getCatalog().begin(), i->getCatalog().end());
+ }
+ // for each file
+ for (const auto& manifestName : manifests) {
+ // find the first (if any) segment we are missing
+ shared_ptr<Name> manifestSegmentName = findManifestSegmentToDownload(manifestName);
+ if (nullptr != manifestSegmentName) {
+ manifestNames.push_back(*manifestSegmentName);
+ }
+ }
+}
+
+void
+TorrentManager::findDataPacketsToDownload(const Name& manifestName, std::vector<Name>& packetNames)
+{
+ auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
+ [&manifestName](const FileManifest& m) {
+ return m.name().getSubName(0, m.name().size()
+ - 1).isPrefixOf(manifestName);
+ });
+
+ for (auto j = manifest_it; j != m_fileManifests.end(); j++) {
+ auto& fileState = m_fileStates[j->getFullName()];
+ for (size_t dataNum = 0; dataNum < j->catalog().size(); ++dataNum) {
+ if (!fileState.second[dataNum]) {
+ packetNames.push_back(j->catalog()[dataNum]);
+ }
+ }
+
+ // check that the next manifest in the vector refers to the next segment of the same file
+ if ((j + 1) != m_fileManifests.end() && (j+1)->file_name() != manifest_it->file_name()) {
+ break;
+ }
+ }
+}
+
+void
+TorrentManager::findAllMissingDataPackets(std::vector<Name>& packetNames)
+{
+ for (auto j = m_fileManifests.begin(); j != m_fileManifests.end(); j++) {
+ auto& fileState = m_fileStates[j->getFullName()];
+ for (auto i = j->catalog().begin(); i != j->catalog().end(); i++) {
+ auto dataNum = i->get(i->size() - 2).toSequenceNumber();
+ if (!fileState.second[dataNum]) {
+ packetNames.push_back(*i);
+ }
+ }
+ }
+}
+
} // end ntorrent
} // end ndn
diff --git a/src/torrent-manager.hpp b/src/torrent-manager.hpp
index a96f21b..28053f2 100644
--- a/src/torrent-manager.hpp
+++ b/src/torrent-manager.hpp
@@ -54,6 +54,7 @@
public:
typedef std::function<void(const ndn::Name&)> DataReceivedCallback;
typedef std::function<void(const std::vector<ndn::Name>&)> ManifestReceivedCallback;
+ typedef std::function<void(const std::vector<ndn::Name>&)> TorrentFileReceivedCallback;
typedef std::function<void(const ndn::Name&, const std::string&)> FailedCallback;
/*
@@ -114,7 +115,7 @@
*/
void
downloadTorrentFile(const std::string& path,
- DataReceivedCallback onSuccess,
+ TorrentFileReceivedCallback onSuccess,
FailedCallback onFailed);
/*
@@ -131,10 +132,11 @@
* This method provides non-blocking downloading of all the file manifest segments
*
*/
- void download_file_manifest(const Name& manifestName,
- const std::string& path,
- ManifestReceivedCallback onSuccess,
- FailedCallback onFailed);
+ void
+ download_file_manifest(const Name& manifestName,
+ const std::string& path,
+ ManifestReceivedCallback onSuccess,
+ FailedCallback onFailed);
/*
* @brief Download a data packet
@@ -147,12 +149,14 @@
* This method writes the downloaded data packet to m_dataPath on disk
*
*/
- void download_data_packet(const Name& packetName,
- DataReceivedCallback onSuccess,
- FailedCallback onFailed);
+ void
+ download_data_packet(const Name& packetName,
+ DataReceivedCallback onSuccess,
+ FailedCallback onFailed);
// Seed the specified 'data' to the network.
- void seed(const Data& data) const;
+ void
+ seed(const Data& data) const;
protected:
/*
@@ -162,7 +166,8 @@
* otherwise. Behavior is undefined unless the corresponding file manifest has already been
* downloaded.
*/
- bool writeData(const Data& packet);
+ bool
+ writeData(const Data& packet);
/*
* \brief Write the @p segment torrent segment to disk at the specified path.
@@ -172,7 +177,8 @@
* otherwise. Behavior is undefined unless @segment is a correct segment for the torrent file of
* this manager and @p path is the directory used for all segments of this torrent file.
*/
- bool writeTorrentSegment(const TorrentFile& segment, const std::string& path);
+ bool
+ writeTorrentSegment(const TorrentFile& segment, const std::string& path);
/*
* \brief Write the @p manifest file manifest to disk at the specified @p path.
@@ -183,7 +189,8 @@
* torrent file of this manager and @p path is the directory used for all file manifests of this
* torrent file.
*/
- bool writeFileManifest(const FileManifest& manifest, const std::string& path);
+ bool
+ writeFileManifest(const FileManifest& manifest, const std::string& path);
/*
* \brief Download the segments of the torrent file
@@ -201,12 +208,13 @@
* specified when async is false
*
*/
- void downloadTorrentFileSegment(const ndn::Name& name,
- const std::string& path,
- std::shared_ptr<std::vector<Name>> manifestNames,
- bool async,
- DataReceivedCallback onSuccess,
- FailedCallback onFailed);
+ void
+ downloadTorrentFileSegment(const ndn::Name& name,
+ const std::string& path,
+ std::shared_ptr<std::vector<Name>> manifestNames,
+ bool async,
+ TorrentFileReceivedCallback onSuccess,
+ FailedCallback onFailed);
/*
* \brief Download the segments of a file manifest
@@ -218,11 +226,12 @@
* @param onFailed Callback to be called when we fail to download a file manifest segment
*
*/
- void downloadFileManifestSegment(const Name& manifestName,
- const std::string& path,
- std::shared_ptr<std::vector<Name>> packetNames,
- TorrentManager::ManifestReceivedCallback onSuccess,
- TorrentManager::FailedCallback onFailed);
+ void
+ downloadFileManifestSegment(const Name& manifestName,
+ const std::string& path,
+ std::shared_ptr<std::vector<Name>> packetNames,
+ ManifestReceivedCallback onSuccess,
+ FailedCallback onFailed);
enum {
// Number of times to retry if a routable prefix fails to retrieve data
@@ -245,6 +254,67 @@
// The path to the location on disk of the Data packet for this manager
std::string m_dataPath;
+protected:
+ /*
+ * \brief Find the torrent file segment that we should download
+ * (either we have nothing or we have them all)
+ * @return A shared_ptr to the name of the segment to download or
+ * nullptr if we have all the segments
+ *
+ */
+ shared_ptr<Name>
+ findTorrentFileSegmentToDownload();
+
+ /*
+ * \brief Given a file manifest segment name, find the next file manifest segment
+ * that we should download
+ * @param manifestName The name of the file manifest segment that we want to download
+ * @return A shared_ptr to the name of the segment to download or
+ * nullptr if we have all the segments
+ *
+ */
+ shared_ptr<Name>
+ findManifestSegmentToDownload(const Name& manifestName);
+
+ /*
+ * \brief Given a data packet name, find whether we have already downloaded this packet
+ * @param dataName The name of the data packet to download
+ * @return True if we have already downloaded this packet, false otherwise
+ *
+ */
+ bool
+ dataAlreadyDownloaded(const Name& dataName);
+
+ /*
+ * \brief Find the segments of all the file manifests that we are missing
+ * @param manifestNames The name of the manifest file segments to download (currently missing)
+ * This parameter is used as an output vector of names
+ */
+ void
+ findFileManifestsToDownload(std::vector<Name>& manifestNames);
+
+ /*
+ * \brief Find the names of the data packets of a file manifest that we are currently missing
+ * @param manifestName The name of the manifest
+ * @param packetNames The name of the data packets to be downloaded
+ * (used as an output vector of names)
+ *
+ * No matter what segment of a manifest file the manifestName parameter might refer to, the
+ * missing data packets starting from the first segment of this manifest file would be returned
+ *
+ */
+ void
+ findDataPacketsToDownload(const Name& manifestName, std::vector<Name>& packetNames);
+
+ /*
+ * \brief Find all the data packets that we are currently missing
+ * @param packetNames The name of the data packets to be downloaded
+ * (used as an output vector of names)
+ *
+ */
+ void
+ findAllMissingDataPackets(std::vector<Name>& packetNames);
+
private:
shared_ptr<Interest>
createInterest(Name name);