blob: 8e525e0ec9c5abc97dd65612f9871d5cb5a16f27 [file] [log] [blame]
Mickey Sweatt527b0492016-03-02 11:07:48 -08001#include "torrent-manager.hpp"
2
3#include "file-manifest.hpp"
4#include "torrent-file.hpp"
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -07005#include "util/io-util.hpp"
Mickey Sweatt617d2d42016-04-25 22:02:08 -07006#include "util/logging.hpp"
Mickey Sweatt527b0492016-03-02 11:07:48 -08007
8#include <boost/filesystem.hpp>
9#include <boost/filesystem/fstream.hpp>
10
11#include <ndn-cxx/data.hpp>
12#include <ndn-cxx/security/key-chain.hpp>
13#include <ndn-cxx/security/signing-helpers.hpp>
14#include <ndn-cxx/util/io.hpp>
15
16#include <set>
17#include <string>
18#include <unordered_map>
19#include <vector>
20
21namespace fs = boost::filesystem;
22
23using std::string;
24using std::vector;
25
Mickey Sweatt527b0492016-03-02 11:07:48 -080026namespace ndn {
27namespace ntorrent {
28
Mickey Sweatt527b0492016-03-02 11:07:48 -080029static vector<TorrentFile>
30intializeTorrentSegments(const string& torrentFilePath, const Name& initialSegmentName)
31{
32 security::KeyChain key_chain;
33 Name currSegmentFullName = initialSegmentName;
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -070034 vector<TorrentFile> torrentSegments = IoUtil::load_directory<TorrentFile>(torrentFilePath);
35
Mickey Sweatt527b0492016-03-02 11:07:48 -080036 // Starting with the initial segment name, verify the names, loading next name from torrentSegment
37 for (auto it = torrentSegments.begin(); it != torrentSegments.end(); ++it) {
38 TorrentFile& segment = *it;
39 key_chain.sign(segment, signingWithSha256());
40 if (segment.getFullName() != currSegmentFullName) {
41 vector<TorrentFile> correctSegments(torrentSegments.begin(), it);
42 torrentSegments.swap(correctSegments);
43 break;
44 }
45 // load the next full name
46 if (nullptr == segment.getTorrentFilePtr()) {
47 break;
48 }
49 currSegmentFullName = *segment.getTorrentFilePtr();
50 }
51 return torrentSegments;
52}
53
54static vector<FileManifest>
Mickey Sweatte908a5c2016-04-08 14:10:45 -070055intializeFileManifests(const string& manifestPath, const vector<TorrentFile>& torrentSegments)
Mickey Sweatt527b0492016-03-02 11:07:48 -080056{
57 security::KeyChain key_chain;
58
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -070059 vector<FileManifest> manifests = IoUtil::load_directory<FileManifest>(manifestPath);
Mickey Sweatte908a5c2016-04-08 14:10:45 -070060 if (manifests.empty()) {
61 return manifests;
62 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -070063
Mickey Sweatt527b0492016-03-02 11:07:48 -080064 // sign the manifests
65 std::for_each(manifests.begin(), manifests.end(),
66 [&key_chain](FileManifest& m){
67 key_chain.sign(m,signingWithSha256());
68 });
69
Mickey Sweatte908a5c2016-04-08 14:10:45 -070070 // put all names of initial manifests from the valid torrent files into a set
71 std::vector<ndn::Name> validInitialManifestNames;
Mickey Sweatt527b0492016-03-02 11:07:48 -080072 for (const auto& segment : torrentSegments) {
73 const auto& catalog = segment.getCatalog();
Mickey Sweatte908a5c2016-04-08 14:10:45 -070074 validInitialManifestNames.insert(validInitialManifestNames.end(),
75 catalog.begin(),
76 catalog.end());
Mickey Sweatt527b0492016-03-02 11:07:48 -080077 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -070078 auto manifest_it = manifests.begin();
79 std::vector<FileManifest> output;
80 output.reserve(manifests.size());
Mickey Sweatt527b0492016-03-02 11:07:48 -080081
Mickey Sweatte908a5c2016-04-08 14:10:45 -070082 for (auto& initialName : validInitialManifestNames) {
83 // starting from the initial segment
84 auto& validName = initialName;
85 if (manifests.end() == manifest_it) {
86 break;
87 }
88 auto fileName = manifest_it->file_name();
89 // sequential collect all valid segments
90 while (manifest_it != manifests.end() && manifest_it->getFullName() == validName) {
91 output.push_back(*manifest_it);
92 if (manifest_it->submanifest_ptr() != nullptr) {
93 validName = *manifest_it->submanifest_ptr();
94 ++manifest_it;
Mickey Sweatt527b0492016-03-02 11:07:48 -080095 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -070096 else {
97 ++manifest_it;
98 break;
99 }
100 }
101 // skip the remain segments for this file (all invalid)
102 while (manifests.end() != manifest_it && manifest_it->file_name() == fileName) {
103 ++manifest_it;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800104 }
105 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700106 return output;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800107}
108
109static vector<Data>
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700110initializeDataPackets(const string& filePath,
111 const FileManifest manifest,
112 size_t subManifestSize)
Mickey Sweatt527b0492016-03-02 11:07:48 -0800113{
114 vector<Data> packets;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700115 auto subManifestNum = manifest.submanifest_number();
Mickey Sweatt527b0492016-03-02 11:07:48 -0800116
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700117 packets = IoUtil::packetize_file(filePath,
118 manifest.name(),
119 manifest.data_packet_size(),
120 subManifestSize,
121 subManifestNum);
Mickey Sweatt527b0492016-03-02 11:07:48 -0800122
123 auto catalog = manifest.catalog();
Mickey Sweatt527b0492016-03-02 11:07:48 -0800124 // Filter out invalid packet names
125 std::remove_if(packets.begin(), packets.end(),
126 [&packets, &catalog](const Data& p) {
127 return catalog.end() == std::find(catalog.begin(),
128 catalog.end(),
129 p.getFullName());
130 });
131 return packets;
132}
133
Mickey Sweattafda1f12016-04-04 17:15:11 -0700134static std::pair<std::shared_ptr<fs::fstream>, std::vector<bool>>
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700135initializeFileState(const string& dataPath,
136 const FileManifest& manifest,
137 size_t subManifestSize)
Mickey Sweattafda1f12016-04-04 17:15:11 -0700138{
139 // construct the file name
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700140 auto fileName = manifest.file_name();
Mickey Sweattafda1f12016-04-04 17:15:11 -0700141 auto filePath = dataPath + fileName;
142 vector<bool> fileBitMap(manifest.catalog().size());
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700143 // if the file does not exist, create an empty placeholder (otherwise cannot set read-bit)
144 if (!fs::exists(filePath)) {
145 fs::ofstream fs(filePath);
146 fs << "";
Mickey Sweattafda1f12016-04-04 17:15:11 -0700147 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700148 auto s = std::make_shared<fs::fstream>(filePath,
149 fs::fstream::out
150 | fs::fstream::binary
151 | fs::fstream::in);
152 if (!*s) {
153 BOOST_THROW_EXCEPTION(io::Error("Cannot open: " + filePath));
154 }
155 auto start_offset = manifest.submanifest_number() * subManifestSize * manifest.data_packet_size();
156 s->seekg(start_offset);
157 s->seekp(start_offset);
Mickey Sweattafda1f12016-04-04 17:15:11 -0700158 return std::make_pair(s, fileBitMap);
159}
160
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700161//==================================================================================================
162// TorrentManager Implementation
163//==================================================================================================
164
Mickey Sweatt527b0492016-03-02 11:07:48 -0800165void TorrentManager::Initialize()
166{
spirosmastorakis4ff8c872016-04-14 09:51:38 -0700167 // initialize the update handler
168
169 // figure out the name of the torrent
170 Name torrentName;
171 if (m_torrentFileName.get(m_torrentFileName.size() - 2).isSequenceNumber()) {
172 torrentName = m_torrentFileName.getSubName(1, m_torrentFileName.size() - 4);
173 }
174 else {
175 torrentName = m_torrentFileName.getSubName(1, m_torrentFileName.size() - 3);
176 }
177
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700178 // TODO(spyros) Get update manager working
Mickey Sweatt67133bf2016-04-25 12:37:35 -0700179 // m_updateHandler = make_shared<UpdateHandler>(torrentName, m_keyChain,
180 // make_shared<StatsTable>(m_statsTable), m_face);
spirosmastorakis4ff8c872016-04-14 09:51:38 -0700181
Mickey Sweatt527b0492016-03-02 11:07:48 -0800182 // .../<torrent_name>/torrent-file/<implicit_digest>
183 string dataPath = ".appdata/" + m_torrentFileName.get(-3).toUri();
184 string manifestPath = dataPath +"/manifests";
185 string torrentFilePath = dataPath +"/torrent_files";
186
187 // get the torrent file segments and manifests that we have.
188 if (!fs::exists(torrentFilePath)) {
189 return;
190 }
191 m_torrentSegments = intializeTorrentSegments(torrentFilePath, m_torrentFileName);
192 if (m_torrentSegments.empty()) {
193 return;
194 }
195 m_fileManifests = intializeFileManifests(manifestPath, m_torrentSegments);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700196
197 // get the submanifest sizes
Mickey Sweatt527b0492016-03-02 11:07:48 -0800198 for (const auto& m : m_fileManifests) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700199 if (m.submanifest_number() == 0) {
200 auto manifestFileName = m.file_name();
201 m_subManifestSizes[manifestFileName] = m.catalog().size();
Mickey Sweatt527b0492016-03-02 11:07:48 -0800202 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700203 }
204
205 for (const auto& m : m_fileManifests) {
Mickey Sweatt527b0492016-03-02 11:07:48 -0800206 // construct the file name
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700207 auto fileName = m.file_name();
Mickey Sweattafda1f12016-04-04 17:15:11 -0700208 fs::path filePath = m_dataPath + fileName;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800209 // If there are any valid packets, add corresponding state to manager
Mickey Sweattafda1f12016-04-04 17:15:11 -0700210 if (!fs::exists(filePath)) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700211 if (!fs::exists(filePath.parent_path())) {
212 boost::filesystem::create_directories(filePath.parent_path());
213 }
Mickey Sweattafda1f12016-04-04 17:15:11 -0700214 continue;
215 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700216 auto packets = initializeDataPackets(filePath.string(), m, m_subManifestSizes[m.file_name()]);
Mickey Sweatt527b0492016-03-02 11:07:48 -0800217 if (!packets.empty()) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700218 m_fileStates[m.getFullName()] = initializeFileState(m_dataPath,
219 m,
220 m_subManifestSizes[m.file_name()]);
Mickey Sweattafda1f12016-04-04 17:15:11 -0700221 auto& fileBitMap = m_fileStates[m.getFullName()].second;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800222 auto read_it = packets.begin();
223 size_t i = 0;
Mickey Sweattafda1f12016-04-04 17:15:11 -0700224 for (auto name : m.catalog()) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700225 if (read_it == packets.end()) {
226 break;
227 }
Mickey Sweatt527b0492016-03-02 11:07:48 -0800228 if (name == read_it->getFullName()) {
229 ++read_it;
Mickey Sweattafda1f12016-04-04 17:15:11 -0700230 fileBitMap[i] = true;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800231 }
232 ++i;
233 }
234 for (const auto& d : packets) {
235 seed(d);
236 }
Mickey Sweatt527b0492016-03-02 11:07:48 -0800237 }
238 }
239 for (const auto& t : m_torrentSegments) {
240 seed(t);
241 }
242 for (const auto& m : m_fileManifests) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700243 seed(m);
Mickey Sweatt527b0492016-03-02 11:07:48 -0800244 }
Mickey Sweattafda1f12016-04-04 17:15:11 -0700245}
Mickey Sweatt527b0492016-03-02 11:07:48 -0800246
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700247shared_ptr<Name>
248TorrentManager::findTorrentFileSegmentToDownload() const
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700249{
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700250 // if we have no segments
251 if (m_torrentSegments.empty()) {
252 return make_shared<Name>(m_torrentFileName);
253 }
254 // otherwise just return the next segment ptr of the last segment we have
255 return m_torrentSegments.back().getTorrentFilePtr();
256}
257
258shared_ptr<Name>
259TorrentManager::findManifestSegmentToDownload(const Name& manifestName) const
260{
261 //sequentially find whether we have downloaded any segments of this manifest file
262 Name manifestPrefix = manifestName.getSubName(0, manifestName.size() - 2);
263 auto it = std::find_if(m_fileManifests.rbegin(), m_fileManifests.rend(),
264 [&manifestPrefix] (const FileManifest& f) {
265 return manifestPrefix.isPrefixOf(f.getName());
266 });
267
268 // if we do not have any segments of the file manifest
269 if (it == m_fileManifests.rend()) {
270 return make_shared<Name>(manifestName);
271 }
272
273 // if we already have the requested segment of the file manifest
274 if (it->submanifest_number() >= manifestName.get(manifestName.size() - 2).toSequenceNumber()) {
275 return it->submanifest_ptr();
276 }
277 // if we do not have the requested segment
278 else {
279 return make_shared<Name>(manifestName);
280 }
281}
282
283void
284TorrentManager::findFileManifestsToDownload(std::vector<Name>& manifestNames) const
285{
286 std::vector<Name> manifests;
287 // insert the first segment name of all the file manifests to the vector
288 for (auto i = m_torrentSegments.begin(); i != m_torrentSegments.end(); i++) {
289 manifests.insert(manifests.end(), i->getCatalog().begin(), i->getCatalog().end());
290 }
291 // for each file
292 for (const auto& manifestName : manifests) {
293 // find the first (if any) segment we are missing
294 shared_ptr<Name> manifestSegmentName = findManifestSegmentToDownload(manifestName);
295 if (nullptr != manifestSegmentName) {
296 manifestNames.push_back(*manifestSegmentName);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700297 }
298 }
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700299}
300
301bool
302TorrentManager::hasDataPacket(const Name& dataName) const
303{
304
305 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
306 [&dataName](const FileManifest& m) {
307 return m.getName().isPrefixOf(dataName);
308 });
309
310 // if we do not have the file manifest, just return false
311 if (manifest_it == m_fileManifests.end()) {
312 return false;
313 }
314
315 // find the pair of (std::shared_ptr<fs::fstream>, std::vector<bool>)
316 // that corresponds to the specific submanifest
317 auto fileState_it = m_fileStates.find(manifest_it->getFullName());
318 if (m_fileStates.end() != fileState_it) {
319 const auto& fileState = fileState_it->second;
320 auto dataNum = dataName.get(dataName.size() - 2).toSequenceNumber();
321 // find whether we have the requested packet from the bitmap
322 return fileState.second[dataNum];
323 }
324 return false;
325}
326
327void
328TorrentManager::findDataPacketsToDownload(const Name& manifestName, std::vector<Name>& packetNames) const
329{
330 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
331 [&manifestName](const FileManifest& m) {
332 return m.name().getSubName(0, m.name().size()
333 - 1).isPrefixOf(manifestName);
334 });
335
336 for (auto j = manifest_it; j != m_fileManifests.end(); j++) {
337 auto& fileState = m_fileStates[j->getFullName()];
338 for (size_t dataNum = 0; dataNum < j->catalog().size(); ++dataNum) {
339 if (!fileState.second[dataNum]) {
340 packetNames.push_back(j->catalog()[dataNum]);
341 }
342 }
343
344 // check that the next manifest in the vector refers to the next segment of the same file
345 if ((j + 1) != m_fileManifests.end() && (j+1)->file_name() != manifest_it->file_name()) {
346 break;
347 }
348 }
349}
350
351void
352TorrentManager::findAllMissingDataPackets(std::vector<Name>& packetNames) const
353{
354 for (auto j = m_fileManifests.begin(); j != m_fileManifests.end(); ++j) {
355 auto fileState_it = m_fileStates.find(j->getFullName());
356 // if we have no packets from this file
357 if (m_fileStates.end() == fileState_it) {
358 packetNames.reserve(packetNames.size() + j->catalog().size());
359 packetNames.insert(packetNames.end(), j->catalog().begin(), j->catalog().end());
360 }
361 // find the packets that we are missing
362 else {
363 const auto &fileState = fileState_it->second;
364 for (auto i = j->catalog().begin(); i != j->catalog().end(); i++) {
365 auto dataNum = i->get(i->size() - 2).toSequenceNumber();
366 if (!fileState.second[dataNum]) {
367 packetNames.push_back(*i);
368 }
369 }
370 }
371 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700372}
373
374void
375TorrentManager::downloadTorrentFileSegment(const ndn::Name& name,
376 const std::string& path,
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700377 TorrentFileReceivedCallback onSuccess,
378 FailedCallback onFailed)
379{
380 shared_ptr<Interest> interest = createInterest(name);
381
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700382 auto dataReceived = [path, onSuccess, onFailed, this]
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700383 (const Interest& interest, const Data& data) {
384 // Stats Table update here...
385 m_stats_table_iter->incrementReceivedData();
386 m_retries = 0;
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700387 std::vector<Name> manifestNames;
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700388 TorrentFile file(data.wireEncode());
389
390 // Write the torrent file segment to disk...
391 if (writeTorrentSegment(file, path)) {
392 // if successfully written, seed this data
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700393 seed(file);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700394 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700395 const std::vector<Name>& manifestCatalog = file.getCatalog();
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700396 manifestNames.insert(manifestNames.end(), manifestCatalog.begin(), manifestCatalog.end());
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700397
398 shared_ptr<Name> nextSegmentPtr = file.getTorrentFilePtr();
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700399 if (onSuccess) {
400 onSuccess(manifestNames);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700401 }
402 if (nextSegmentPtr != nullptr) {
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700403 this->downloadTorrentFileSegment(*nextSegmentPtr, path, onSuccess, onFailed);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700404 }
405 };
406
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700407 auto dataFailed = [path, name, onSuccess, onFailed, this]
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700408 (const Interest& interest) {
409 ++m_retries;
410 if (m_retries >= MAX_NUM_OF_RETRIES) {
411 ++m_stats_table_iter;
412 if (m_stats_table_iter == m_statsTable.end()) {
413 m_stats_table_iter = m_statsTable.begin();
414 }
415 }
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700416 if (onFailed) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700417 onFailed(interest.getName(), "Unknown error");
418 }
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700419 this->downloadTorrentFileSegment(name, path, onSuccess, onFailed);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700420 };
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700421 LOG_TRACE << "Sending: " << *interest << std::endl;
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700422 m_face->expressInterest(*interest, dataReceived, dataFailed);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700423}
424
425void
426TorrentManager::downloadTorrentFile(const std::string& path,
427 TorrentFileReceivedCallback onSuccess,
428 FailedCallback onFailed)
429{
430 shared_ptr<Name> searchRes = this->findTorrentFileSegmentToDownload();
431 auto manifestNames = make_shared<std::vector<Name>>();
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700432 if (searchRes != nullptr) {
433 this->downloadTorrentFileSegment(*searchRes, path, onSuccess, onFailed);
434 }
435 else {
436 std::vector<Name> manifests;
437 findFileManifestsToDownload(manifests);
438 if (onSuccess) {
439 onSuccess(manifests);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700440 }
441 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700442}
443
444void
445TorrentManager::download_file_manifest(const Name& manifestName,
446 const std::string& path,
447 TorrentManager::ManifestReceivedCallback onSuccess,
448 TorrentManager::FailedCallback onFailed)
449{
450 shared_ptr<Name> searchRes = findManifestSegmentToDownload(manifestName);
451 auto packetNames = make_shared<std::vector<Name>>();
452 if (searchRes == nullptr) {
453 this->findDataPacketsToDownload(manifestName, *packetNames);
454 onSuccess(*packetNames);
455 return;
456 }
457 this->downloadFileManifestSegment(*searchRes, path, packetNames, onSuccess, onFailed);
458}
459
460void
461TorrentManager::download_data_packet(const Name& packetName,
462 DataReceivedCallback onSuccess,
463 FailedCallback onFailed)
464{
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700465 if (this->hasDataPacket(packetName)) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700466 onSuccess(packetName);
467 return;
468 }
469
470 shared_ptr<Interest> interest = this->createInterest(packetName);
471
472 auto dataReceived = [onSuccess, onFailed, this]
473 (const Interest& interest, const Data& data) {
474 // Write data to disk...
475 if(writeData(data)) {
476 seed(data);
477 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700478 // Stats Table update here...
479 m_stats_table_iter->incrementReceivedData();
480 m_retries = 0;
481 onSuccess(data.getName());
482 };
483 auto dataFailed = [onFailed, this]
484 (const Interest& interest) {
485 m_retries++;
486 if (m_retries >= MAX_NUM_OF_RETRIES) {
487 m_stats_table_iter++;
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700488 if (m_stats_table_iter == m_statsTable.end()) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700489 m_stats_table_iter = m_statsTable.begin();
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700490 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700491 }
492 onFailed(interest.getName(), "Unknown failure");
493 };
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700494 LOG_TRACE << "Sending: " << *interest << std::endl;
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700495 m_face->expressInterest(*interest, dataReceived, dataFailed);
496}
497
498void TorrentManager::seed(const Data& data) {
499 m_face->setInterestFilter(data.getFullName(),
500 bind(&TorrentManager::onInterestReceived, this, _1, _2),
501 RegisterPrefixSuccessCallback(),
502 bind(&TorrentManager::onRegisterFailed, this, _1, _2));
503}
504
505// = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
506// Protected Helpers
507// = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
508
Mickey Sweattafda1f12016-04-04 17:15:11 -0700509bool TorrentManager::writeData(const Data& packet)
510{
511 // find correct manifest
512 const auto& packetName = packet.getName();
513 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
514 [&packetName](const FileManifest& m) {
515 return m.getName().isPrefixOf(packetName);
516 });
517 if (m_fileManifests.end() == manifest_it) {
518 return false;
519 }
520 // get file state out
521 auto& fileState = m_fileStates[manifest_it->getFullName()];
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700522
Mickey Sweattafda1f12016-04-04 17:15:11 -0700523 // if there is no open stream to the file
524 if (nullptr == fileState.first) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700525 fs::path filePath = m_dataPath + manifest_it->file_name();
526 if (!fs::exists(filePath)) {
527 fs::create_directories(filePath.parent_path());
528 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700529 fileState = initializeFileState(m_dataPath,
530 *manifest_it,
531 m_subManifestSizes[manifest_it->file_name()]);
Mickey Sweattafda1f12016-04-04 17:15:11 -0700532 }
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700533 auto packetNum = packetName.get(packetName.size() - 1).toSequenceNumber();
Mickey Sweattafda1f12016-04-04 17:15:11 -0700534 // if we already have the packet, do not rewrite it.
535 if (fileState.second[packetNum]) {
536 return false;
537 }
Mickey Sweattafda1f12016-04-04 17:15:11 -0700538 // write data to disk
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700539 // TODO(msweatt) Fix this once code is merged
540 auto subManifestSize = m_subManifestSizes[manifest_it->file_name()];
541 if (IoUtil::writeData(packet, *manifest_it, subManifestSize, *fileState.first)) {
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700542 fileState.first->flush();
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700543 // update bitmap
544 fileState.second[packetNum] = true;
545 return true;
Mickey Sweattafda1f12016-04-04 17:15:11 -0700546 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700547 return false;
Mickey Sweatt527b0492016-03-02 11:07:48 -0800548}
549
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700550bool
551TorrentManager::writeTorrentSegment(const TorrentFile& segment, const std::string& path)
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700552{
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700553 // validate the torrent
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700554 auto torrentPrefix = m_torrentFileName.getSubName(0, m_torrentFileName.size() - 1);
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700555 // check if we already have it
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700556 if (torrentPrefix.isPrefixOf(segment.getName()) &&
557 m_torrentSegments.end() == std::find(m_torrentSegments.begin(), m_torrentSegments.end(),
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700558 segment))
559 {
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700560 if(IoUtil::writeTorrentSegment(segment, path)) {
561 auto it = std::find_if(m_torrentSegments.begin(), m_torrentSegments.end(),
562 [&segment](const TorrentFile& t){
563 return segment.getSegmentNumber() < t.getSegmentNumber() ;
564 });
565 m_torrentSegments.insert(it, segment);
566 return true;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700567 }
568 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700569 return false;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700570}
571
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700572
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700573bool TorrentManager::writeFileManifest(const FileManifest& manifest, const std::string& path)
574{
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700575 if (m_fileManifests.end() == std::find(m_fileManifests.begin(), m_fileManifests.end(),
576 manifest))
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700577 {
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700578 // update the state of the manager
579 if (0 == manifest.submanifest_number()) {
580 m_subManifestSizes[manifest.file_name()] = manifest.catalog().size();
581 }
582 if(IoUtil::writeFileManifest(manifest, path)) {
583 // add to collection
584 auto it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
585 [&manifest](const FileManifest& m){
586 return m.file_name() > manifest.file_name()
587 || (m.file_name() == manifest.file_name()
588 && (m.submanifest_number() > manifest.submanifest_number()));
589 });
590 m_fileManifests.insert(it, manifest);
591 return true;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700592 }
593 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700594 return false;
Mickey Sweatt599bfef2016-04-05 19:11:20 -0700595}
596
spirosmastorakisa46eee42016-04-05 14:24:45 -0700597void
598TorrentManager::downloadFileManifestSegment(const Name& manifestName,
599 const std::string& path,
600 std::shared_ptr<std::vector<Name>> packetNames,
601 TorrentManager::ManifestReceivedCallback onSuccess,
602 TorrentManager::FailedCallback onFailed)
603{
604 shared_ptr<Interest> interest = this->createInterest(manifestName);
605
606 auto dataReceived = [packetNames, path, onSuccess, onFailed, this]
607 (const Interest& interest, const Data& data) {
608 // Stats Table update here...
609 m_stats_table_iter->incrementReceivedData();
610 m_retries = 0;
611
612 FileManifest file(data.wireEncode());
613
614 // Write the file manifest segment to disk...
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700615 if(writeFileManifest(file, path)) {
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700616 seed(file);
617 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700618 else {
619 onFailed(interest.getName(), "Write Failed");
620 }
spirosmastorakisa46eee42016-04-05 14:24:45 -0700621
622 const std::vector<Name>& packetsCatalog = file.catalog();
623 packetNames->insert(packetNames->end(), packetsCatalog.begin(), packetsCatalog.end());
624 shared_ptr<Name> nextSegmentPtr = file.submanifest_ptr();
625 if (nextSegmentPtr != nullptr) {
626 this->downloadFileManifestSegment(*nextSegmentPtr, path, packetNames, onSuccess, onFailed);
627 }
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700628 else {
spirosmastorakisa46eee42016-04-05 14:24:45 -0700629 onSuccess(*packetNames);
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700630 }
spirosmastorakisa46eee42016-04-05 14:24:45 -0700631 };
632
633 auto dataFailed = [packetNames, path, manifestName, onFailed, this]
634 (const Interest& interest) {
635 m_retries++;
636 if (m_retries >= MAX_NUM_OF_RETRIES) {
637 m_stats_table_iter++;
638 if (m_stats_table_iter == m_statsTable.end())
639 m_stats_table_iter = m_statsTable.begin();
640 }
641 onFailed(interest.getName(), "Unknown failure");
642 };
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700643 LOG_TRACE << "Sending: " << *interest << std::endl;
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700644 m_face->expressInterest(*interest, dataReceived, dataFailed);
spirosmastorakisa46eee42016-04-05 14:24:45 -0700645}
646
647void
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700648TorrentManager::onInterestReceived(const InterestFilter& filter, const Interest& interest)
spirosmastorakisa46eee42016-04-05 14:24:45 -0700649{
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700650 // handle if it is a torrent-file
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700651 LOG_TRACE << "Interest Recevied: " << interest << std::endl;
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700652 const auto& interestName = interest.getName();
653 std::shared_ptr<Data> data = nullptr;
654 auto cmp = [&interestName](const Data& t){return t.getFullName() == interestName;};
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700655 // determine if it is torrent file (that we have)
656 auto torrent_it = std::find_if(m_torrentSegments.begin(), m_torrentSegments.end(), cmp);
657 if (m_torrentSegments.end() != torrent_it) {
658 data = std::make_shared<Data>(*torrent_it);
spirosmastorakis50642f82016-04-08 12:11:18 -0700659 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700660 else {
661 // determine if it is manifest (that we have)
662 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(), cmp);
663 if (m_fileManifests.end() != manifest_it) {
664 data = std::make_shared<Data>(*manifest_it) ;
spirosmastorakisa46eee42016-04-05 14:24:45 -0700665 }
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700666 else {
667 // determine if it is data packet (that we have)
668 auto manifestName = interestName.getSubName(0, interestName.size() - 2);
669 auto map_it = std::find_if(m_fileStates.begin(), m_fileStates.end(),
670 [&manifestName](const std::pair<Name,
671 std::pair<std::shared_ptr<fs::fstream>,
672 std::vector<bool>>>& kv){
673 return manifestName.isPrefixOf(kv.first);
674 });
675 if (m_fileStates.end() != map_it) {
676 auto packetName = interestName.getSubName(0, interestName.size() - 1);
677 // get out the bitmap to be sure we have the packet
678 auto& fileState = map_it->second;
679 const auto &bitmap = fileState.second;
680 auto packetNum = packetName.get(packetName.size() - 1).toSequenceNumber();
681 if (bitmap[packetNum]) {
682 // get the manifest
683 auto manifest_it = std::find_if(m_fileManifests.begin(), m_fileManifests.end(),
684 [&manifestName](const FileManifest& m) {
685 return manifestName.isPrefixOf(m.name());
686 });
687 auto manifestFileName = manifest_it->file_name();
688 auto filePath = m_dataPath + manifestFileName;
689 // TODO(msweatt) Explore why fileState stream does not work
690 fs::fstream is (filePath, fs::fstream::in | fs::fstream::binary);
Mickey Sweattfcbfb3d2016-04-13 17:05:17 -0700691 data = IoUtil::readDataPacket(interestName,
692 *manifest_it,
693 m_subManifestSizes[manifestFileName],
694 is);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700695 }
696 }
697 }
698 }
699 if (nullptr != data) {
700 m_face->put(*data);
701 }
702 else {
703 // TODO(msweatt) NACK
Mickey Sweatt617d2d42016-04-25 22:02:08 -0700704 LOG_ERROR << "NACK: " << interest << std::endl;
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700705 }
706 return;
spirosmastorakisa46eee42016-04-05 14:24:45 -0700707}
708
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700709void
710TorrentManager::onRegisterFailed(const Name& prefix, const std::string& reason)
spirosmastorakisa46eee42016-04-05 14:24:45 -0700711{
Mickey Sweatt617d2d42016-04-25 22:02:08 -0700712 LOG_ERROR << "ERROR: Failed to register prefix \""
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700713 << prefix << "\" in local hub's daemon (" << reason << ")"
714 << std::endl;
715 m_face->shutdown();
spirosmastorakisa46eee42016-04-05 14:24:45 -0700716}
717
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700718shared_ptr<Interest>
719TorrentManager::createInterest(Name name)
720{
721 shared_ptr<Interest> interest = make_shared<Interest>(name);
722 interest->setInterestLifetime(time::milliseconds(2000));
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700723
724 // Select routable prefix
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700725 // TODO(spyros) Fix links
726 // Link link(name, { {1, m_stats_table_iter->getRecordName()} });
727 // m_keyChain->sign(link, signingWithSha256());
728 // Block linkWire = link.wireEncode();
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700729
730 // Stats Table update here...
731 m_stats_table_iter->incrementSentInterests();
732
733 m_sortingCounter++;
734 if (m_sortingCounter >= SORTING_INTERVAL) {
spirosmastorakis4ff8c872016-04-14 09:51:38 -0700735 // Use the sorting interval to send out "ALIVE" Interests as well
736 // check whether we should send out an "ALIVE" Interest
Mickey Sweatt67133bf2016-04-25 12:37:35 -0700737 // if (m_updateHandler->needsUpdate()) {
738 // m_updateHandler->sendAliveInterest(m_stats_table_iter);
739 // }
spirosmastorakis4ff8c872016-04-14 09:51:38 -0700740 // Do the actual sorting related stuff
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700741 m_sortingCounter = 0;
742 m_statsTable.sort();
743 m_stats_table_iter = m_statsTable.begin();
744 m_retries = 0;
745 }
746
Mickey Sweatt15dde2d2016-04-28 23:42:45 -0700747 // interest->setLink(linkWire);
Mickey Sweatte908a5c2016-04-08 14:10:45 -0700748
749 return interest;
750}
751
Mickey Sweatt527b0492016-03-02 11:07:48 -0800752} // end ntorrent
spirosmastorakisfd334462016-04-18 15:48:31 -0700753} // end ndn