blob: c6780e28db65caec404aa0e19cc2b1b3b15eb6ac [file] [log] [blame]
Mickey Sweatt527b0492016-03-02 11:07:48 -08001#include "torrent-manager.hpp"
2
3#include "file-manifest.hpp"
4#include "torrent-file.hpp"
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -07005#include "util/io-util.hpp"
Mickey Sweatt527b0492016-03-02 11:07:48 -08006
7#include <boost/filesystem.hpp>
8#include <boost/filesystem/fstream.hpp>
9
10#include <ndn-cxx/data.hpp>
11#include <ndn-cxx/security/key-chain.hpp>
12#include <ndn-cxx/security/signing-helpers.hpp>
13#include <ndn-cxx/util/io.hpp>
14
15#include <set>
16#include <string>
17#include <unordered_map>
18#include <vector>
19
20namespace fs = boost::filesystem;
21
22using std::string;
23using std::vector;
24
Mickey Sweatt527b0492016-03-02 11:07:48 -080025namespace ndn {
26namespace ntorrent {
27
Mickey Sweatt527b0492016-03-02 11:07:48 -080028static vector<TorrentFile>
29intializeTorrentSegments(const string& torrentFilePath, const Name& initialSegmentName)
30{
31 security::KeyChain key_chain;
32 Name currSegmentFullName = initialSegmentName;
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -070033 vector<TorrentFile> torrentSegments = IoUtil::load_directory<TorrentFile>(torrentFilePath);
34
Mickey Sweatt527b0492016-03-02 11:07:48 -080035 // Starting with the initial segment name, verify the names, loading next name from torrentSegment
36 for (auto it = torrentSegments.begin(); it != torrentSegments.end(); ++it) {
37 TorrentFile& segment = *it;
38 key_chain.sign(segment, signingWithSha256());
39 if (segment.getFullName() != currSegmentFullName) {
40 vector<TorrentFile> correctSegments(torrentSegments.begin(), it);
41 torrentSegments.swap(correctSegments);
42 break;
43 }
44 // load the next full name
45 if (nullptr == segment.getTorrentFilePtr()) {
46 break;
47 }
48 currSegmentFullName = *segment.getTorrentFilePtr();
49 }
50 return torrentSegments;
51}
52
53static vector<FileManifest>
Mickey Sweatte908a5c2016-04-08 14:10:45 -070054intializeFileManifests(const string& manifestPath, const vector<TorrentFile>& torrentSegments)
Mickey Sweatt527b0492016-03-02 11:07:48 -080055{
56 security::KeyChain key_chain;
57
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -070058 vector<FileManifest> manifests = IoUtil::load_directory<FileManifest>(manifestPath);
Mickey Sweatte908a5c2016-04-08 14:10:45 -070059 if (manifests.empty()) {
60 return manifests;
61 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -070062
Mickey Sweatt527b0492016-03-02 11:07:48 -080063 // sign the manifests
64 std::for_each(manifests.begin(), manifests.end(),
65 [&key_chain](FileManifest& m){
66 key_chain.sign(m,signingWithSha256());
67 });
68
Mickey Sweatte908a5c2016-04-08 14:10:45 -070069 // put all names of initial manifests from the valid torrent files into a set
70 std::vector<ndn::Name> validInitialManifestNames;
Mickey Sweatt527b0492016-03-02 11:07:48 -080071 for (const auto& segment : torrentSegments) {
72 const auto& catalog = segment.getCatalog();
Mickey Sweatte908a5c2016-04-08 14:10:45 -070073 validInitialManifestNames.insert(validInitialManifestNames.end(),
74 catalog.begin(),
75 catalog.end());
Mickey Sweatt527b0492016-03-02 11:07:48 -080076 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -070077 auto manifest_it = manifests.begin();
78 std::vector<FileManifest> output;
79 output.reserve(manifests.size());
Mickey Sweatt527b0492016-03-02 11:07:48 -080080
Mickey Sweatte908a5c2016-04-08 14:10:45 -070081 for (auto& initialName : validInitialManifestNames) {
82 // starting from the initial segment
83 auto& validName = initialName;
84 if (manifests.end() == manifest_it) {
85 break;
86 }
87 auto fileName = manifest_it->file_name();
88 // sequential collect all valid segments
89 while (manifest_it != manifests.end() && manifest_it->getFullName() == validName) {
90 output.push_back(*manifest_it);
91 if (manifest_it->submanifest_ptr() != nullptr) {
92 validName = *manifest_it->submanifest_ptr();
93 ++manifest_it;
Mickey Sweatt527b0492016-03-02 11:07:48 -080094 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -070095 else {
96 ++manifest_it;
97 break;
98 }
99 }
100 // skip the remain segments for this file (all invalid)
101 while (manifests.end() != manifest_it && manifest_it->file_name() == fileName) {
102 ++manifest_it;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800103 }
104 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700105 return output;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800106}
107
108static vector<Data>
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700109initializeDataPackets(const string& filePath,
110 const FileManifest manifest,
111 size_t subManifestSize)
Mickey Sweatt527b0492016-03-02 11:07:48 -0800112{
113 vector<Data> packets;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700114 auto subManifestNum = manifest.submanifest_number();
Mickey Sweatt527b0492016-03-02 11:07:48 -0800115
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700116 packets = IoUtil::packetize_file(filePath,
117 manifest.name(),
118 manifest.data_packet_size(),
119 subManifestSize,
120 subManifestNum);
Mickey Sweatt527b0492016-03-02 11:07:48 -0800121
122 auto catalog = manifest.catalog();
Mickey Sweatt527b0492016-03-02 11:07:48 -0800123 // Filter out invalid packet names
124 std::remove_if(packets.begin(), packets.end(),
125 [&packets, &catalog](const Data& p) {
126 return catalog.end() == std::find(catalog.begin(),
127 catalog.end(),
128 p.getFullName());
129 });
130 return packets;
131}
132
Mickey Sweattafda1f12016-04-04 17:15:11 -0700133static std::pair<std::shared_ptr<fs::fstream>, std::vector<bool>>
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700134initializeFileState(const string& dataPath,
135 const FileManifest& manifest,
136 size_t subManifestSize)
Mickey Sweattafda1f12016-04-04 17:15:11 -0700137{
138 // construct the file name
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700139 auto fileName = manifest.file_name();
Mickey Sweattafda1f12016-04-04 17:15:11 -0700140 auto filePath = dataPath + fileName;
141 vector<bool> fileBitMap(manifest.catalog().size());
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700142 // if the file does not exist, create an empty placeholder (otherwise cannot set read-bit)
143 if (!fs::exists(filePath)) {
144 fs::ofstream fs(filePath);
145 fs << "";
Mickey Sweattafda1f12016-04-04 17:15:11 -0700146 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700147 auto s = std::make_shared<fs::fstream>(filePath,
148 fs::fstream::out
149 | fs::fstream::binary
150 | fs::fstream::in);
151 if (!*s) {
152 BOOST_THROW_EXCEPTION(io::Error("Cannot open: " + filePath));
153 }
154 auto start_offset = manifest.submanifest_number() * subManifestSize * manifest.data_packet_size();
155 s->seekg(start_offset);
156 s->seekp(start_offset);
Mickey Sweattafda1f12016-04-04 17:15:11 -0700157 return std::make_pair(s, fileBitMap);
158}
159
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700160//==================================================================================================
161// TorrentManager Implementation
162//==================================================================================================
163
Mickey Sweatt527b0492016-03-02 11:07:48 -0800164void TorrentManager::Initialize()
165{
166 // .../<torrent_name>/torrent-file/<implicit_digest>
167 string dataPath = ".appdata/" + m_torrentFileName.get(-3).toUri();
168 string manifestPath = dataPath +"/manifests";
169 string torrentFilePath = dataPath +"/torrent_files";
170
171 // get the torrent file segments and manifests that we have.
172 if (!fs::exists(torrentFilePath)) {
173 return;
174 }
175 m_torrentSegments = intializeTorrentSegments(torrentFilePath, m_torrentFileName);
176 if (m_torrentSegments.empty()) {
177 return;
178 }
179 m_fileManifests = intializeFileManifests(manifestPath, m_torrentSegments);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700180
181 // get the submanifest sizes
Mickey Sweatt527b0492016-03-02 11:07:48 -0800182 for (const auto& m : m_fileManifests) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700183 if (m.submanifest_number() == 0) {
184 auto manifestFileName = m.file_name();
185 m_subManifestSizes[manifestFileName] = m.catalog().size();
Mickey Sweatt527b0492016-03-02 11:07:48 -0800186 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700187 }
188
189 for (const auto& m : m_fileManifests) {
Mickey Sweatt527b0492016-03-02 11:07:48 -0800190 // construct the file name
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700191 auto fileName = m.file_name();
Mickey Sweattafda1f12016-04-04 17:15:11 -0700192 fs::path filePath = m_dataPath + fileName;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800193 // If there are any valid packets, add corresponding state to manager
Mickey Sweattafda1f12016-04-04 17:15:11 -0700194 if (!fs::exists(filePath)) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700195 if (!fs::exists(filePath.parent_path())) {
196 boost::filesystem::create_directories(filePath.parent_path());
197 }
Mickey Sweattafda1f12016-04-04 17:15:11 -0700198 continue;
199 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700200 auto packets = initializeDataPackets(filePath.string(), m, m_subManifestSizes[m.file_name()]);
Mickey Sweatt527b0492016-03-02 11:07:48 -0800201 if (!packets.empty()) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700202 m_fileStates[m.getFullName()] = initializeFileState(m_dataPath,
203 m,
204 m_subManifestSizes[m.file_name()]);
Mickey Sweattafda1f12016-04-04 17:15:11 -0700205 auto& fileBitMap = m_fileStates[m.getFullName()].second;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800206 auto read_it = packets.begin();
207 size_t i = 0;
Mickey Sweattafda1f12016-04-04 17:15:11 -0700208 for (auto name : m.catalog()) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700209 if (read_it == packets.end()) {
210 break;
211 }
Mickey Sweatt527b0492016-03-02 11:07:48 -0800212 if (name == read_it->getFullName()) {
213 ++read_it;
Mickey Sweattafda1f12016-04-04 17:15:11 -0700214 fileBitMap[i] = true;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800215 }
216 ++i;
217 }
218 for (const auto& d : packets) {
219 seed(d);
220 }
Mickey Sweatt527b0492016-03-02 11:07:48 -0800221 }
222 }
223 for (const auto& t : m_torrentSegments) {
224 seed(t);
225 }
226 for (const auto& m : m_fileManifests) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700227 seed(m);
Mickey Sweatt527b0492016-03-02 11:07:48 -0800228 }
Mickey Sweattafda1f12016-04-04 17:15:11 -0700229}
Mickey Sweatt527b0492016-03-02 11:07:48 -0800230
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700231std::vector<Name>
232TorrentManager::downloadTorrentFile(const std::string& path)
233{
234 shared_ptr<Name> searchRes = this->findTorrentFileSegmentToDownload();
235 auto manifestNames = make_shared<std::vector<Name>>();
236 if (searchRes == nullptr) {
237 this->findFileManifestsToDownload(*manifestNames);
238 if (manifestNames->empty()) {
239 auto packetNames = make_shared<std::vector<Name>>();
240 this->findAllMissingDataPackets(*packetNames);
241 return *packetNames;
242 }
243 else {
244 return *manifestNames;
245 }
246 }
247 this->downloadTorrentFileSegment(m_torrentFileName, path, manifestNames,
248 false, {}, {});
249 return *manifestNames;
250}
251
252void
253TorrentManager::downloadTorrentFileSegment(const ndn::Name& name,
254 const std::string& path,
255 std::shared_ptr<std::vector<Name>> manifestNames,
256 bool async,
257 TorrentFileReceivedCallback onSuccess,
258 FailedCallback onFailed)
259{
260 shared_ptr<Interest> interest = createInterest(name);
261
262 auto dataReceived = [manifestNames, path, async, onSuccess, onFailed, this]
263 (const Interest& interest, const Data& data) {
264 // Stats Table update here...
265 m_stats_table_iter->incrementReceivedData();
266 m_retries = 0;
267
268 if (async) {
269 manifestNames->clear();
270 }
271
272 TorrentFile file(data.wireEncode());
273
274 // Write the torrent file segment to disk...
275 if (writeTorrentSegment(file, path)) {
276 // if successfully written, seed this data
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700277 seed(file);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700278 }
279
280 const std::vector<Name>& manifestCatalog = file.getCatalog();
281 manifestNames->insert(manifestNames->end(), manifestCatalog.begin(), manifestCatalog.end());
282
283 shared_ptr<Name> nextSegmentPtr = file.getTorrentFilePtr();
284
285 if (async) {
286 onSuccess(*manifestNames);
287 }
288 if (nextSegmentPtr != nullptr) {
289 this->downloadTorrentFileSegment(*nextSegmentPtr, path, manifestNames,
290 async, onSuccess, onFailed);
291 }
292 };
293
294 auto dataFailed = [manifestNames, path, name, async, onSuccess, onFailed, this]
295 (const Interest& interest) {
296 ++m_retries;
297 if (m_retries >= MAX_NUM_OF_RETRIES) {
298 ++m_stats_table_iter;
299 if (m_stats_table_iter == m_statsTable.end()) {
300 m_stats_table_iter = m_statsTable.begin();
301 }
302 }
303 if (async) {
304 onFailed(interest.getName(), "Unknown error");
305 }
306 this->downloadTorrentFileSegment(name, path, manifestNames, async, onSuccess, onFailed);
307 };
308
309 m_face->expressInterest(*interest, dataReceived, dataFailed);
310
311 if (!async) {
312 m_face->processEvents();
313 }
314}
315
316void
317TorrentManager::downloadTorrentFile(const std::string& path,
318 TorrentFileReceivedCallback onSuccess,
319 FailedCallback onFailed)
320{
321 shared_ptr<Name> searchRes = this->findTorrentFileSegmentToDownload();
322 auto manifestNames = make_shared<std::vector<Name>>();
323 if (searchRes == nullptr) {
324 this->findFileManifestsToDownload(*manifestNames);
325 if (manifestNames->empty()) {
326 auto packetNames = make_shared<std::vector<Name>>();
327 this->findAllMissingDataPackets(*packetNames);
328 onSuccess(*packetNames);
329 return;
330 }
331 else {
332 onSuccess(*manifestNames);
333 return;
334 }
335 }
336 this->downloadTorrentFileSegment(*searchRes, path, manifestNames,
337 true, onSuccess, onFailed);
338}
339
340void
341TorrentManager::download_file_manifest(const Name& manifestName,
342 const std::string& path,
343 TorrentManager::ManifestReceivedCallback onSuccess,
344 TorrentManager::FailedCallback onFailed)
345{
346 shared_ptr<Name> searchRes = findManifestSegmentToDownload(manifestName);
347 auto packetNames = make_shared<std::vector<Name>>();
348 if (searchRes == nullptr) {
349 this->findDataPacketsToDownload(manifestName, *packetNames);
350 onSuccess(*packetNames);
351 return;
352 }
353 this->downloadFileManifestSegment(*searchRes, path, packetNames, onSuccess, onFailed);
354}
355
356void
357TorrentManager::download_data_packet(const Name& packetName,
358 DataReceivedCallback onSuccess,
359 FailedCallback onFailed)
360{
361 if (this->dataAlreadyDownloaded(packetName)) {
362 onSuccess(packetName);
363 return;
364 }
365
366 shared_ptr<Interest> interest = this->createInterest(packetName);
367
368 auto dataReceived = [onSuccess, onFailed, this]
369 (const Interest& interest, const Data& data) {
370 // Write data to disk...
371 if(writeData(data)) {
372 seed(data);
373 }
374
375 // Stats Table update here...
376 m_stats_table_iter->incrementReceivedData();
377 m_retries = 0;
378 onSuccess(data.getName());
379 };
380 auto dataFailed = [onFailed, this]
381 (const Interest& interest) {
382 m_retries++;
383 if (m_retries >= MAX_NUM_OF_RETRIES) {
384 m_stats_table_iter++;
385 if (m_stats_table_iter == m_statsTable.end())
386 m_stats_table_iter = m_statsTable.begin();
387 }
388 onFailed(interest.getName(), "Unknown failure");
389 };
390
391 m_face->expressInterest(*interest, dataReceived, dataFailed);
392}
393
394void TorrentManager::seed(const Data& data) {
395 m_face->setInterestFilter(data.getFullName(),
396 bind(&TorrentManager::onInterestReceived, this, _1, _2),
397 RegisterPrefixSuccessCallback(),
398 bind(&TorrentManager::onRegisterFailed, this, _1, _2));
399}
400
401// = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
402// Protected Helpers
403// = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
404
Mickey Sweattafda1f12016-04-04 17:15:11 -0700405bool TorrentManager::writeData(const Data& packet)
406{
407 // find correct manifest
408 const auto& packetName = packet.getName();
409 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
410 [&packetName](const FileManifest& m) {
411 return m.getName().isPrefixOf(packetName);
412 });
413 if (m_fileManifests.end() == manifest_it) {
414 return false;
415 }
416 // get file state out
417 auto& fileState = m_fileStates[manifest_it->getFullName()];
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700418
Mickey Sweattafda1f12016-04-04 17:15:11 -0700419 // if there is no open stream to the file
420 if (nullptr == fileState.first) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700421 fs::path filePath = m_dataPath + manifest_it->file_name();
422 if (!fs::exists(filePath)) {
423 fs::create_directories(filePath.parent_path());
424 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700425 fileState = initializeFileState(m_dataPath,
426 *manifest_it,
427 m_subManifestSizes[manifest_it->file_name()]);
Mickey Sweattafda1f12016-04-04 17:15:11 -0700428 }
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700429 auto packetNum = packetName.get(packetName.size() - 1).toSequenceNumber();
Mickey Sweattafda1f12016-04-04 17:15:11 -0700430 // if we already have the packet, do not rewrite it.
431 if (fileState.second[packetNum]) {
432 return false;
433 }
Mickey Sweattafda1f12016-04-04 17:15:11 -0700434 // write data to disk
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700435 // TODO(msweatt) Fix this once code is merged
436 auto subManifestSize = m_subManifestSizes[manifest_it->file_name()];
437 if (IoUtil::writeData(packet, *manifest_it, subManifestSize, *fileState.first)) {
438 // update bitmap
439 fileState.second[packetNum] = true;
440 return true;
Mickey Sweattafda1f12016-04-04 17:15:11 -0700441 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700442 return false;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800443}
444
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700445bool
446TorrentManager::writeTorrentSegment(const TorrentFile& segment, const std::string& path)
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700447{
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700448 // validate the torrent
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700449 auto torrentPrefix = m_torrentFileName.getSubName(0, m_torrentFileName.size() - 1);
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700450 // check if we already have it
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700451 if (torrentPrefix.isPrefixOf(segment.getName()) &&
452 m_torrentSegments.end() == std::find(m_torrentSegments.begin(), m_torrentSegments.end(),
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700453 segment))
454 {
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700455 if(IoUtil::writeTorrentSegment(segment, path)) {
456 auto it = std::find_if(m_torrentSegments.begin(), m_torrentSegments.end(),
457 [&segment](const TorrentFile& t){
458 return segment.getSegmentNumber() < t.getSegmentNumber() ;
459 });
460 m_torrentSegments.insert(it, segment);
461 return true;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700462 }
463 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700464 return false;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700465}
466
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700467
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700468bool TorrentManager::writeFileManifest(const FileManifest& manifest, const std::string& path)
469{
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700470 if (m_fileManifests.end() == std::find(m_fileManifests.begin(), m_fileManifests.end(),
471 manifest))
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700472 {
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700473 // update the state of the manager
474 if (0 == manifest.submanifest_number()) {
475 m_subManifestSizes[manifest.file_name()] = manifest.catalog().size();
476 }
477 if(IoUtil::writeFileManifest(manifest, path)) {
478 // add to collection
479 auto it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
480 [&manifest](const FileManifest& m){
481 return m.file_name() > manifest.file_name()
482 || (m.file_name() == manifest.file_name()
483 && (m.submanifest_number() > manifest.submanifest_number()));
484 });
485 m_fileManifests.insert(it, manifest);
486 return true;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700487 }
488 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700489 return false;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700490}
491
spirosmastorakisa46eee42016-04-05 14:24:45 -0700492void
493TorrentManager::downloadFileManifestSegment(const Name& manifestName,
494 const std::string& path,
495 std::shared_ptr<std::vector<Name>> packetNames,
496 TorrentManager::ManifestReceivedCallback onSuccess,
497 TorrentManager::FailedCallback onFailed)
498{
499 shared_ptr<Interest> interest = this->createInterest(manifestName);
500
501 auto dataReceived = [packetNames, path, onSuccess, onFailed, this]
502 (const Interest& interest, const Data& data) {
503 // Stats Table update here...
504 m_stats_table_iter->incrementReceivedData();
505 m_retries = 0;
506
507 FileManifest file(data.wireEncode());
508
509 // Write the file manifest segment to disk...
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700510 if(writeFileManifest(file, path)) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700511 seed(file);
512 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700513 else {
514 onFailed(interest.getName(), "Write Failed");
515 }
spirosmastorakisa46eee42016-04-05 14:24:45 -0700516
517 const std::vector<Name>& packetsCatalog = file.catalog();
518 packetNames->insert(packetNames->end(), packetsCatalog.begin(), packetsCatalog.end());
519 shared_ptr<Name> nextSegmentPtr = file.submanifest_ptr();
520 if (nextSegmentPtr != nullptr) {
521 this->downloadFileManifestSegment(*nextSegmentPtr, path, packetNames, onSuccess, onFailed);
522 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700523 else {
spirosmastorakisa46eee42016-04-05 14:24:45 -0700524 onSuccess(*packetNames);
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700525 }
spirosmastorakisa46eee42016-04-05 14:24:45 -0700526 };
527
528 auto dataFailed = [packetNames, path, manifestName, onFailed, this]
529 (const Interest& interest) {
530 m_retries++;
531 if (m_retries >= MAX_NUM_OF_RETRIES) {
532 m_stats_table_iter++;
533 if (m_stats_table_iter == m_statsTable.end())
534 m_stats_table_iter = m_statsTable.begin();
535 }
536 onFailed(interest.getName(), "Unknown failure");
537 };
538
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700539 m_face->expressInterest(*interest, dataReceived, dataFailed);
spirosmastorakisa46eee42016-04-05 14:24:45 -0700540}
541
542void
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700543TorrentManager::onInterestReceived(const InterestFilter& filter, const Interest& interest)
spirosmastorakisa46eee42016-04-05 14:24:45 -0700544{
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700545 // handle if it is a torrent-file
546 const auto& interestName = interest.getName();
547 std::shared_ptr<Data> data = nullptr;
548 auto cmp = [&interestName](const Data& t){return t.getFullName() == interestName;};
549
550 // determine if it is torrent file (that we have)
551 auto torrent_it = std::find_if(m_torrentSegments.begin(), m_torrentSegments.end(), cmp);
552 if (m_torrentSegments.end() != torrent_it) {
553 data = std::make_shared<Data>(*torrent_it);
spirosmastorakis50642f82016-04-08 12:11:18 -0700554 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700555 else {
556 // determine if it is manifest (that we have)
557 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(), cmp);
558 if (m_fileManifests.end() != manifest_it) {
559 data = std::make_shared<Data>(*manifest_it) ;
spirosmastorakisa46eee42016-04-05 14:24:45 -0700560 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700561 else {
562 // determine if it is data packet (that we have)
563 auto manifestName = interestName.getSubName(0, interestName.size() - 2);
564 auto map_it = std::find_if(m_fileStates.begin(), m_fileStates.end(),
565 [&manifestName](const std::pair<Name,
566 std::pair<std::shared_ptr<fs::fstream>,
567 std::vector<bool>>>& kv){
568 return manifestName.isPrefixOf(kv.first);
569 });
570 if (m_fileStates.end() != map_it) {
571 auto packetName = interestName.getSubName(0, interestName.size() - 1);
572 // get out the bitmap to be sure we have the packet
573 auto& fileState = map_it->second;
574 const auto &bitmap = fileState.second;
575 auto packetNum = packetName.get(packetName.size() - 1).toSequenceNumber();
576 if (bitmap[packetNum]) {
577 // get the manifest
578 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
579 [&manifestName](const FileManifest& m) {
580 return manifestName.isPrefixOf(m.name());
581 });
582 auto manifestFileName = manifest_it->file_name();
583 auto filePath = m_dataPath + manifestFileName;
584 // TODO(msweatt) Explore why fileState stream does not work
585 fs::fstream is (filePath, fs::fstream::in | fs::fstream::binary);
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700586 data = IoUtil::readDataPacket(interestName,
587 *manifest_it,
588 m_subManifestSizes[manifestFileName],
589 is);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700590 }
591 }
592 }
593 }
594 if (nullptr != data) {
595 m_face->put(*data);
596 }
597 else {
598 // TODO(msweatt) NACK
599 std::cerr << "NACK: " << interest << std::endl;
600 }
601 return;
spirosmastorakisa46eee42016-04-05 14:24:45 -0700602}
603
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700604void
605TorrentManager::onRegisterFailed(const Name& prefix, const std::string& reason)
spirosmastorakisa46eee42016-04-05 14:24:45 -0700606{
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700607 std::cerr << "ERROR: Failed to register prefix \""
608 << prefix << "\" in local hub's daemon (" << reason << ")"
609 << std::endl;
610 m_face->shutdown();
spirosmastorakisa46eee42016-04-05 14:24:45 -0700611}
612
spirosmastorakis50642f82016-04-08 12:11:18 -0700613shared_ptr<Name>
614TorrentManager::findTorrentFileSegmentToDownload()
615{
616 // if we have no segments
617 if (m_torrentSegments.empty()) {
618 return make_shared<Name>(m_torrentFileName);
619 }
620 // otherwise just return the next segment ptr of the last segment we have
621 return m_torrentSegments.back().getTorrentFilePtr();
622}
623
624shared_ptr<Name>
625TorrentManager::findManifestSegmentToDownload(const Name& manifestName)
626{
627 //sequentially find whether we have downloaded any segments of this manifest file
628 Name manifestPrefix = manifestName.getSubName(0, manifestName.size() - 2);
629 auto it = std::find_if(m_fileManifests.rbegin(), m_fileManifests.rend(),
630 [&manifestPrefix] (const FileManifest& f) {
631 return manifestPrefix.isPrefixOf(f.getName());
632 });
633
634 // if we do not have any segments of the file manifest
635 if (it == m_fileManifests.rend()) {
636 return make_shared<Name>(manifestName);
637 }
638
639 // if we already have the requested segment of the file manifest
640 if (it->submanifest_number() >= manifestName.get(manifestName.size() - 2).toSequenceNumber()) {
641 return it->submanifest_ptr();
642 }
643 // if we do not have the requested segment
644 else {
645 return make_shared<Name>(manifestName);
646 }
647}
648
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700649void
650TorrentManager::findFileManifestsToDownload(std::vector<Name>& manifestNames)
651{
652 std::vector<Name> manifests;
653 // insert the first segment name of all the file manifests to the vector
654 for (auto i = m_torrentSegments.begin(); i != m_torrentSegments.end(); i++) {
655 manifests.insert(manifests.end(), i->getCatalog().begin(), i->getCatalog().end());
656 }
657 // for each file
658 for (const auto& manifestName : manifests) {
659 // find the first (if any) segment we are missing
660 shared_ptr<Name> manifestSegmentName = findManifestSegmentToDownload(manifestName);
661 if (nullptr != manifestSegmentName) {
662 manifestNames.push_back(*manifestSegmentName);
663 }
664 }
665}
666
spirosmastorakis50642f82016-04-08 12:11:18 -0700667bool
668TorrentManager::dataAlreadyDownloaded(const Name& dataName)
669{
670
671 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
672 [&dataName](const FileManifest& m) {
673 return m.getName().isPrefixOf(dataName);
674 });
675
676 // if we do not have the file manifest, just return false
677 if (manifest_it == m_fileManifests.end()) {
678 return false;
679 }
680
681 // find the pair of (std::shared_ptr<fs::fstream>, std::vector<bool>)
682 // that corresponds to the specific submanifest
683 auto& fileState = m_fileStates[manifest_it->getFullName()];
684
685 auto dataNum = dataName.get(dataName.size() - 2).toSequenceNumber();
686
687 // find whether we have the requested packet from the bitmap
688 return fileState.second[dataNum];
689}
690
691void
spirosmastorakis50642f82016-04-08 12:11:18 -0700692TorrentManager::findDataPacketsToDownload(const Name& manifestName, std::vector<Name>& packetNames)
693{
694 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
695 [&manifestName](const FileManifest& m) {
696 return m.name().getSubName(0, m.name().size()
697 - 1).isPrefixOf(manifestName);
698 });
699
700 for (auto j = manifest_it; j != m_fileManifests.end(); j++) {
701 auto& fileState = m_fileStates[j->getFullName()];
702 for (size_t dataNum = 0; dataNum < j->catalog().size(); ++dataNum) {
703 if (!fileState.second[dataNum]) {
704 packetNames.push_back(j->catalog()[dataNum]);
705 }
706 }
707
708 // check that the next manifest in the vector refers to the next segment of the same file
709 if ((j + 1) != m_fileManifests.end() && (j+1)->file_name() != manifest_it->file_name()) {
710 break;
711 }
712 }
713}
714
715void
716TorrentManager::findAllMissingDataPackets(std::vector<Name>& packetNames)
717{
718 for (auto j = m_fileManifests.begin(); j != m_fileManifests.end(); j++) {
719 auto& fileState = m_fileStates[j->getFullName()];
720 for (auto i = j->catalog().begin(); i != j->catalog().end(); i++) {
721 auto dataNum = i->get(i->size() - 2).toSequenceNumber();
722 if (!fileState.second[dataNum]) {
723 packetNames.push_back(*i);
724 }
725 }
726 }
727}
728
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700729shared_ptr<Interest>
730TorrentManager::createInterest(Name name)
731{
732 shared_ptr<Interest> interest = make_shared<Interest>(name);
733 interest->setInterestLifetime(time::milliseconds(2000));
734 interest->setMustBeFresh(true);
735
736 // Select routable prefix
737 Link link(name, { {1, m_stats_table_iter->getRecordName()} });
738 m_keyChain->sign(link, signingWithSha256());
739 Block linkWire = link.wireEncode();
740
741 // Stats Table update here...
742 m_stats_table_iter->incrementSentInterests();
743
744 m_sortingCounter++;
745 if (m_sortingCounter >= SORTING_INTERVAL) {
746 m_sortingCounter = 0;
747 m_statsTable.sort();
748 m_stats_table_iter = m_statsTable.begin();
749 m_retries = 0;
750 }
751
752 interest->setLink(linkWire);
753
754 return interest;
755}
756
Mickey Sweatt527b0492016-03-02 11:07:48 -0800757} // end ntorrent
spirosmastorakisfd334462016-04-18 15:48:31 -0700758} // end ndn