Merge remote branch 'origin/bug_348_dvdburn_residenttask' into experimental
[enigma2.git] / lib / dvb / tstools.cpp
1 #define _ISOC99_SOURCE /* for llabs */
2 #include <lib/dvb/tstools.h>
3 #include <lib/base/eerror.h>
4 #include <unistd.h>
5 #include <fcntl.h>
6
7 #include <stdio.h>
8
9 eDVBTSTools::eDVBTSTools()
10         :m_file_lock(true)
11 {
12         m_pid = -1;
13         m_maxrange = 256*1024;
14         
15         m_begin_valid = 0;
16         m_end_valid = 0;
17         
18         m_use_streaminfo = 0;
19         m_samples_taken = 0;
20         
21         m_last_filelength = 0;
22         
23         m_futile = 0;
24 }
25
26 eDVBTSTools::~eDVBTSTools()
27 {
28         closeFile();
29 }
30
31 int eDVBTSTools::openFile(const char *filename, int nostreaminfo)
32 {
33         closeFile();
34         
35         if (!nostreaminfo)
36         {
37                 eDebug("loading streaminfo for %s", filename);
38                 m_streaminfo.load(filename);
39         }
40         
41         if (!m_streaminfo.empty())
42                 m_use_streaminfo = 1;
43         else
44         {
45 //              eDebug("no recorded stream information available");
46                 m_use_streaminfo = 0;
47         }
48         
49         m_samples_taken = 0;
50
51         eSingleLocker l(m_file_lock);
52         if (m_file.open(filename, 1) < 0)
53                 return -1;
54         return 0;
55 }
56
57 void eDVBTSTools::closeFile()
58 {
59         eSingleLocker l(m_file_lock);
60         m_file.close();
61 }
62
63 void eDVBTSTools::setSyncPID(int pid)
64 {
65         m_pid = pid;
66 }
67
68 void eDVBTSTools::setSearchRange(int maxrange)
69 {
70         m_maxrange = maxrange;
71 }
72
73         /* getPTS extracts a pts value from any PID at a given offset. */
74 int eDVBTSTools::getPTS(off_t &offset, pts_t &pts, int fixed)
75 {
76         if (m_use_streaminfo)
77                 if (!m_streaminfo.getPTS(offset, pts))
78                         return 0;
79         
80         if (!m_file.valid())
81                 return -1;
82
83         offset -= offset % 188;
84
85         eSingleLocker l(m_file_lock);
86         if (m_file.lseek(offset, SEEK_SET) < 0)
87         {
88                 eDebug("lseek failed");
89                 return -1;
90         }
91         
92         int left = m_maxrange;
93         
94         while (left >= 188)
95         {
96                 unsigned char packet[188];
97                 if (m_file.read(packet, 188) != 188)
98                 {
99                         eDebug("read error");
100                         break;
101                 }
102                 left -= 188;
103                 offset += 188;
104                 
105                 if (packet[0] != 0x47)
106                 {
107                         eDebug("resync");
108                         int i = 0;
109                         while (i < 188)
110                         {
111                                 if (packet[i] == 0x47)
112                                         break;
113                                 ++i;
114                         }
115                         offset = m_file.lseek(i - 188, SEEK_CUR);
116                         continue;
117                 }
118                 
119                 int pid = ((packet[1] << 8) | packet[2]) & 0x1FFF;
120                 int pusi = !!(packet[1] & 0x40);
121                 
122 //              printf("PID %04x, PUSI %d\n", pid, pusi);
123
124                 unsigned char *payload;
125                 
126                         /* check for adaption field */
127                 if (packet[3] & 0x20)
128                 {
129                         if (packet[4] >= 183)
130                                 continue;
131                         if (packet[4])
132                         {
133                                 if (packet[5] & 0x10) /* PCR present */
134                                 {
135                                         pts  = ((unsigned long long)(packet[ 6]&0xFF)) << 25;
136                                         pts |= ((unsigned long long)(packet[ 7]&0xFF)) << 17;
137                                         pts |= ((unsigned long long)(packet[ 8]&0xFE)) << 9;
138                                         pts |= ((unsigned long long)(packet[ 9]&0xFF)) << 1;
139                                         pts |= ((unsigned long long)(packet[10]&0x80)) >> 7;
140                                         offset -= 188;
141                                         eDebug("PCR %16llx found at %lld pid %02x (%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x)", pts, offset, pid, packet[0], packet[1], packet[2], packet[3], packet[4], packet[5], packet[6], packet[7], packet[8], packet[9], packet[10]);
142                                         if (fixed && fixupPTS(offset, pts))
143                                                 return -1;
144                                         return 0;
145                                 }
146                         }
147                         payload = packet + packet[4] + 4 + 1;
148                 } else
149                         payload = packet + 4;
150
151 /*              if (m_pid >= 0)
152                         if (pid != m_pid)
153                                 continue; */
154                 if (!pusi)
155                         continue;
156
157                         /* somehow not a startcode. (this is invalid, since pusi was set.) ignore it. */
158                 if (payload[0] || payload[1] || (payload[2] != 1))
159                         continue;
160
161                 if (payload[3] == 0xFD)
162                 { // stream use extension mechanism defined in ISO 13818-1 Amendment 2
163                         if (payload[7] & 1) // PES extension flag
164                         {
165                                 int offs = 0;
166                                 if (payload[7] & 0x80) // pts avail
167                                         offs += 5;
168                                 if (payload[7] & 0x40) // dts avail
169                                         offs += 5;
170                                 if (payload[7] & 0x20) // escr avail
171                                         offs += 6;
172                                 if (payload[7] & 0x10) // es rate
173                                         offs += 3;
174                                 if (payload[7] & 0x8) // dsm trickmode
175                                         offs += 1;
176                                 if (payload[7] & 0x4) // additional copy info
177                                         offs += 1;
178                                 if (payload[7] & 0x2) // crc
179                                         offs += 2;
180                                 if (payload[8] < offs)
181                                         continue;
182                                 uint8_t pef = payload[9+offs++]; // pes extension field
183                                 if (pef & 1) // pes extension flag 2
184                                 {
185                                         if (pef & 0x80) // private data flag
186                                                 offs += 16;
187                                         if (pef & 0x40) // pack header field flag
188                                                 offs += 1;
189                                         if (pef & 0x20) // program packet sequence counter flag
190                                                 offs += 2;
191                                         if (pef & 0x10) // P-STD buffer flag
192                                                 offs += 2;
193                                         if (payload[8] < offs)
194                                                 continue;
195                                         uint8_t stream_id_extension_len = payload[9+offs++] & 0x7F;
196                                         if (stream_id_extension_len >= 1)
197                                         {
198                                                 if (payload[8] < (offs + stream_id_extension_len) )
199                                                         continue;
200                                                 if (payload[9+offs] & 0x80) // stream_id_extension_bit (should not set)
201                                                         continue;
202                                                 switch (payload[9+offs])
203                                                 {
204                                                 case 0x55 ... 0x5f: // VC-1
205                                                         break;
206                                                 case 0x71: // AC3 / DTS
207                                                         break;
208                                                 default:
209                                                         eDebug("skip unknwn stream_id_extension %02x\n", payload[9+offs]);
210                                                         continue;
211                                                 }
212                                         }
213                                         else
214                                                 continue;
215                                 }
216                                 else
217                                         continue;
218                         }
219                         else
220                                 continue;
221                 }
222                         /* drop non-audio, non-video packets because other streams
223                            can be non-compliant.*/
224                 else if (((payload[3] & 0xE0) != 0xC0) &&  // audio
225                         ((payload[3] & 0xF0) != 0xE0)) // video
226                         continue;
227
228                 if (payload[7] & 0x80) /* PTS */
229                 {
230                         pts  = ((unsigned long long)(payload[ 9]&0xE))  << 29;
231                         pts |= ((unsigned long long)(payload[10]&0xFF)) << 22;
232                         pts |= ((unsigned long long)(payload[11]&0xFE)) << 14;
233                         pts |= ((unsigned long long)(payload[12]&0xFF)) << 7;
234                         pts |= ((unsigned long long)(payload[13]&0xFE)) >> 1;
235                         offset -= 188;
236
237                         eDebug("PTS %16llx found at %lld pid %02x stream: %02x", pts, offset, pid, payload[3]);
238
239                                 /* convert to zero-based */
240                         if (fixed && fixupPTS(offset, pts))
241                                 return -1;
242                         return 0;
243                 }
244         }
245         
246         return -1;
247 }
248
249 int eDVBTSTools::fixupPTS(const off_t &offset, pts_t &now)
250 {
251         if (m_use_streaminfo)
252         {
253                 if (!m_streaminfo.fixupPTS(offset, now))
254                         return 0;
255         } else
256         {
257                         /* for the simple case, we assume one epoch, with up to one wrap around in the middle. */
258                 calcBegin();
259                 if (!m_begin_valid)
260                 {       
261                         eDebug("begin not valid, can't fixup");
262                         return -1;
263                 }
264                 
265                 pts_t pos = m_pts_begin;
266                 if ((now < pos) && ((pos - now) < 90000 * 10))
267                 {       
268                         pos = 0;
269                         return 0;
270                 }
271                 
272                 if (now < pos) /* wrap around */
273                         now = now + 0x200000000LL - pos;
274                 else
275                         now -= pos;
276                 return 0;
277         }
278         eDebug("eDVBTSTools::fixupPTS failed!");
279         return -1;
280 }
281
282 int eDVBTSTools::getOffset(off_t &offset, pts_t &pts, int marg)
283 {
284         eDebug("getOffset for pts 0x%llx", pts);
285         if (m_use_streaminfo)
286         {
287                 if (pts >= m_pts_end && marg > 0 && m_end_valid)
288                         offset = m_offset_end;
289                 else
290                         offset = m_streaminfo.getAccessPoint(pts, marg);
291                 return 0;
292         } else
293         {
294                 calcBegin(); calcEnd();
295                 
296                 if (!m_begin_valid)
297                         return -1;
298                 if (!m_end_valid)
299                         return -1;
300
301                 if (!m_samples_taken)
302                         takeSamples();
303                 
304                 if (!m_samples.empty())
305                 {
306                         int maxtries = 5;
307                         pts_t p = -1;
308                         
309                         while (maxtries--)
310                         {
311                                         /* search entry before and after */
312                                 std::map<pts_t, off_t>::const_iterator l = m_samples.lower_bound(pts);
313                                 std::map<pts_t, off_t>::const_iterator u = l;
314
315                                 if (l != m_samples.begin())
316                                         --l;
317                                 
318                                         /* we could have seeked beyond the end */
319                                 if (u == m_samples.end())
320                                 {
321                                                 /* use last segment for interpolation. */
322                                         if (l != m_samples.begin())
323                                         {
324                                                 --u;
325                                                 --l;
326                                         }
327                                 }
328                                         
329                                         /* if we don't have enough points */
330                                 if (u == m_samples.end())
331                                         break;
332                                 
333                                 pts_t pts_diff = u->first - l->first;
334                                 off_t offset_diff = u->second - l->second;
335                                 
336                                 if (offset_diff < 0)
337                                 {
338                                         eDebug("something went wrong when taking samples.");
339                                         m_samples.clear();
340                                         takeSamples();
341                                         continue;
342                                 }
343
344                                 eDebug("using: %llx:%llx -> %llx:%llx", l->first, u->first, l->second, u->second);
345
346                                 int bitrate;
347                                 
348                                 if (pts_diff)
349                                         bitrate = offset_diff * 90000 * 8 / pts_diff;
350                                 else
351                                         bitrate = 0;
352
353                                 offset = l->second;
354                                 offset += ((pts - l->first) * (pts_t)bitrate) / 8ULL / 90000ULL;
355                                 offset -= offset % 188;
356                                 
357                                 p = pts;
358                                 
359                                 if (!takeSample(offset, p))
360                                 {
361                                         int diff = (p - pts) / 90;
362                         
363                                         eDebug("calculated diff %d ms", diff);
364                                         if (abs(diff) > 300)
365                                         {
366                                                 eDebug("diff to big, refining");
367                                                 continue;
368                                         }
369                                 } else
370                                         eDebug("no sample taken, refinement not possible.");
371
372                                 break;
373                         }
374                         
375                                 /* if even the first sample couldn't be taken, fall back. */
376                                 /* otherwise, return most refined result. */
377                         if (p != -1)
378                         {
379                                 pts = p;
380                                 eDebug("aborting. Taking %llx as offset for %lld", offset, pts);
381                                 return 0;
382                         }
383                 }
384                 
385                 int bitrate = calcBitrate();
386                 offset = pts * (pts_t)bitrate / 8ULL / 90000ULL;
387                 eDebug("fallback, bitrate=%d, results in %016llx", bitrate, offset);
388                 offset -= offset % 188;
389                 
390                 return 0;
391         }
392 }
393
394 int eDVBTSTools::getNextAccessPoint(pts_t &ts, const pts_t &start, int direction)
395 {
396         if (m_use_streaminfo)
397                 return m_streaminfo.getNextAccessPoint(ts, start, direction);
398         else
399         {
400                 eDebug("can't get next access point without streaminfo");
401                 return -1;
402         }
403 }
404
405 void eDVBTSTools::calcBegin()
406 {
407         if (!m_file.valid())
408                 return;
409
410         if (!(m_begin_valid || m_futile))
411         {
412                 m_offset_begin = 0;
413                 if (!getPTS(m_offset_begin, m_pts_begin))
414                         m_begin_valid = 1;
415                 else
416                         m_futile = 1;
417         }
418 }
419
420 void eDVBTSTools::calcEnd()
421 {
422         if (!m_file.valid())
423                 return;
424
425         eSingleLocker l(m_file_lock);
426         off_t end = m_file.lseek(0, SEEK_END);
427         
428         if (llabs(end - m_last_filelength) > 1*1024*1024)
429         {
430                 m_last_filelength = end;
431                 m_end_valid = 0;
432                 
433                 m_futile = 0;
434 //              eDebug("file size changed, recalc length");
435         }
436         
437         int maxiter = 10;
438         
439         m_offset_end = m_last_filelength;
440         
441         while (!(m_end_valid || m_futile))
442         {
443                 if (!--maxiter)
444                 {
445                         m_futile = 1;
446                         return;
447                 }
448
449                 m_offset_end -= m_maxrange;
450                 if (m_offset_end < 0)
451                         m_offset_end = 0;
452
453                         /* restore offset if getpts fails */
454                 off_t off = m_offset_end;
455
456                 if (!getPTS(m_offset_end, m_pts_end))
457                         m_end_valid = 1;
458                 else
459                         m_offset_end = off;
460
461                 if (!m_offset_end)
462                 {
463                         m_futile = 1;
464                         break;
465                 }
466         }
467 }
468
469 int eDVBTSTools::calcLen(pts_t &len)
470 {
471         calcBegin(); calcEnd();
472         if (!(m_begin_valid && m_end_valid))
473                 return -1;
474         len = m_pts_end - m_pts_begin;
475                 /* wrap around? */
476         if (len < 0)
477                 len += 0x200000000LL;
478         return 0;
479 }
480
481 int eDVBTSTools::calcBitrate()
482 {
483         calcBegin(); calcEnd();
484         if (!(m_begin_valid && m_end_valid))
485                 return -1;
486
487         pts_t len_in_pts = m_pts_end - m_pts_begin;
488
489                 /* wrap around? */
490         if (len_in_pts < 0)
491                 len_in_pts += 0x200000000LL;
492         off_t len_in_bytes = m_offset_end - m_offset_begin;
493         
494         if (!len_in_pts)
495                 return -1;
496         
497         unsigned long long bitrate = len_in_bytes * 90000 * 8 / len_in_pts;
498         if ((bitrate < 10000) || (bitrate > 100000000))
499                 return -1;
500         
501         return bitrate;
502 }
503
504         /* pts, off */
505 void eDVBTSTools::takeSamples()
506 {
507         m_samples_taken = 1;
508         m_samples.clear();
509         pts_t dummy;
510         int retries=2;
511
512         if (calcLen(dummy) == -1)
513                 return;
514         
515         int nr_samples = 30;
516         off_t bytes_per_sample = (m_offset_end - m_offset_begin) / (long long)nr_samples;
517         if (bytes_per_sample < 40*1024*1024)
518                 bytes_per_sample = 40*1024*1024;
519
520         bytes_per_sample -= bytes_per_sample % 188;
521
522         eDebug("samples step %lld, pts begin %llx, pts end %llx, offs begin %lld, offs end %lld:",
523                 bytes_per_sample, m_pts_begin, m_pts_end, m_offset_begin, m_offset_end);
524
525         for (off_t offset = m_offset_begin; offset < m_offset_end;)
526         {
527                 pts_t p;
528                 if (takeSample(offset, p) && retries--)
529                         continue;
530                 retries = 2;
531                 offset += bytes_per_sample;
532         }
533         m_samples[0] = m_offset_begin;
534         m_samples[m_pts_end - m_pts_begin] = m_offset_end;
535 }
536
537         /* returns 0 when a sample was taken. */
538 int eDVBTSTools::takeSample(off_t off, pts_t &p)
539 {
540         off_t offset_org = off;
541
542         if (!eDVBTSTools::getPTS(off, p, 1))
543         {
544                         /* as we are happily mixing PTS and PCR values (no comment, please), we might
545                            end up with some "negative" segments. 
546                            
547                            so check if this new sample is between the previous and the next field*/
548
549                 std::map<pts_t, off_t>::const_iterator l = m_samples.lower_bound(p);
550                 std::map<pts_t, off_t>::const_iterator u = l;
551
552                 if (l != m_samples.begin())
553                 {
554                         --l;
555                         if (u != m_samples.end())
556                         {
557                                 if ((l->second > off) || (u->second < off))
558                                 {
559                                         eDebug("ignoring sample %lld %lld %lld (%llx %llx %llx)",
560                                                 l->second, off, u->second, l->first, p, u->first);
561                                         return 1;
562                                 }
563                         }
564                 }
565
566                 eDebug("adding sample %lld: pts 0x%llx -> pos %lld (diff %lld bytes)", offset_org, p, off, off-offset_org);
567                 m_samples[p] = off;
568                 return 0;
569         }
570         return -1;
571 }
572
573 int eDVBTSTools::findPMT(int &pmt_pid, int &service_id)
574 {
575                 /* FIXME: this will be factored out soon! */
576         if (!m_file.valid())
577         {
578                 eDebug(" file not valid");
579                 return -1;
580         }
581
582         eSingleLocker l(m_file_lock);
583         if (m_file.lseek(0, SEEK_SET) < 0)
584         {
585                 eDebug("seek failed");
586                 return -1;
587         }
588
589         int left = 5*1024*1024;
590         
591         while (left >= 188)
592         {
593                 unsigned char packet[188];
594                 if (m_file.read(packet, 188) != 188)
595                 {
596                         eDebug("read error");
597                         break;
598                 }
599                 left -= 188;
600                 
601                 if (packet[0] != 0x47)
602                 {
603                         int i = 0;
604                         while (i < 188)
605                         {
606                                 if (packet[i] == 0x47)
607                                         break;
608                                 ++i;
609                         }
610                         m_file.lseek(i - 188, SEEK_CUR);
611                         continue;
612                 }
613                 
614                 int pid = ((packet[1] << 8) | packet[2]) & 0x1FFF;
615                 
616                 int pusi = !!(packet[1] & 0x40);
617                 
618                 if (!pusi)
619                         continue;
620                 
621                         /* ok, now we have a PES header or section header*/
622                 unsigned char *sec;
623                 
624                         /* check for adaption field */
625                 if (packet[3] & 0x20)
626                 {
627                         if (packet[4] >= 183)
628                                 continue;
629                         sec = packet + packet[4] + 4 + 1;
630                 } else
631                         sec = packet + 4;
632                 
633                 if (sec[0])     /* table pointer, assumed to be 0 */
634                         continue;
635
636                 if (sec[1] == 0x02) /* program map section */
637                 {
638                         pmt_pid = pid;
639                         service_id = (sec[4] << 8) | sec[5];
640                         return 0;
641                 }
642         }
643         
644         return -1;
645 }
646
647 int eDVBTSTools::findFrame(off_t &_offset, size_t &len, int &direction, int frame_types)
648 {
649         off_t offset = _offset;
650         int nr_frames = 0;
651 //      eDebug("trying to find iFrame at %llx", offset);
652
653         if (!m_use_streaminfo)
654         {
655 //              eDebug("can't get next iframe without streaminfo");
656                 return -1;
657         }
658
659                                 /* let's find the iframe before the given offset */
660         unsigned long long data;
661         
662         if (direction < 0)
663                 offset--;
664
665         while (1)
666         {
667                 if (m_streaminfo.getStructureEntry(offset, data, (direction == 0) ? 1 : 0))
668                 {
669                         eDebug("getting structure info for origin offset failed.");
670                         return -1;
671                 }
672                 if (offset == 0x7fffffffffffffffLL) /* eof */
673                 {
674                         eDebug("reached eof");
675                         return -1;
676                 }
677                         /* data is usually the start code in the lower 8 bit, and the next byte <<8. we extract the picture type from there */
678                         /* we know that we aren't recording startcode 0x09 for mpeg2, so this is safe */
679                         /* TODO: check frame_types */
680                 int is_start = (data & 0xE0FF) == 0x0009; /* H.264 NAL unit access delimiter with I-frame*/
681                 is_start |= (data & 0x3800FF) == 0x080000; /* MPEG2 picture start code with I-frame */
682                 
683                 int is_frame = ((data & 0xFF) == 0x0009) || ((data & 0xFF) == 0x00); /* H.264 UAD or MPEG2 start code */
684                 
685                 if (is_frame)
686                 {
687                         if (direction < 0)
688                                 --nr_frames;
689                         else
690                                 ++nr_frames;
691                 }
692 //              eDebug("%08llx@%llx -> %d, %d", data, offset, is_start, nr_frames);
693                 if (is_start)
694                         break;
695
696                 if (direction == -1)
697                         --offset; /* move to previous entry */
698                 else if (direction == +1)
699                         direction = 0;
700         }
701                         /* let's find the next frame after the given offset */
702         off_t start = offset;
703
704         do {
705                 if (m_streaminfo.getStructureEntry(offset, data, 1))
706                 {
707                         eDebug("get next failed");
708                         return -1;
709                 }
710                 if (offset == 0x7fffffffffffffffLL) /* eof */
711                 {
712                         eDebug("reached eof (while looking for end of iframe)");
713                         return -1;
714                 }
715 //              eDebug("%08llx@%llx (next)", data, offset);
716         } while (((data & 0xFF) != 9) && ((data & 0xFF) != 0x00)); /* next frame */
717
718                         /* align to TS pkt start */
719 //      start = start - (start % 188);
720 //      offset = offset - (offset % 188);
721
722         len = offset - start;
723         _offset = start;
724         direction = nr_frames;
725 //      eDebug("result: offset=%llx, len: %ld", offset, (int)len);
726         return 0;
727 }
728
729 int eDVBTSTools::findNextPicture(off_t &offset, size_t &len, int &distance, int frame_types)
730 {
731         int nr_frames, direction;
732 //      eDebug("trying to move %d frames at %llx", distance, offset);
733         
734         frame_types = frametypeI; /* TODO: intelligent "allow IP frames when not crossing an I-Frame */
735
736         off_t new_offset = offset;
737         size_t new_len = len;
738         int first = 1;
739
740         if (distance > 0) {
741                 direction = 0;
742                 nr_frames = 0;
743         } else {
744                 direction = -1;
745                 nr_frames = -1;
746                 distance = -distance+1;
747         }       
748         while (distance > 0)
749         {
750                 int dir = direction;
751                 if (findFrame(new_offset, new_len, dir, frame_types))
752                 {
753 //                      eDebug("findFrame failed!\n");
754                         return -1;
755                 }
756                 
757                 distance -= abs(dir);
758                 
759 //              eDebug("we moved %d, %d to go frames (now at %llx)", dir, distance, new_offset);
760
761                 if (distance >= 0 || direction == 0)
762                 {
763                         first = 0;
764                         offset = new_offset;
765                         len = new_len;
766                         nr_frames += abs(dir);
767                 } 
768                 else if (first) {
769                         first = 0;
770                         offset = new_offset;
771                         len = new_len;
772                         nr_frames += abs(dir) + distance; // never jump forward during rewind
773                 }
774         }
775
776         distance = (direction < 0) ? -nr_frames : nr_frames;
777 //      eDebug("in total, we moved %d frames", nr_frames);
778
779         return 0;
780 }