optimize epg for less memory consumption,
authorAndreas Monzner <andreas.monzner@multimedia-labs.de>
Mon, 29 Aug 2005 20:50:20 +0000 (20:50 +0000)
committerAndreas Monzner <andreas.monzner@multimedia-labs.de>
Mon, 29 Aug 2005 20:50:20 +0000 (20:50 +0000)
add RESULT return values to lookupEvent,
we need still a python interface for query the epgcache

lib/dvb/epgcache.cpp
lib/dvb/epgcache.h

index 28cd855..568519c 100644 (file)
 #include <lib/base/eerror.h>
 
 int eventData::CacheSize=0;
+descriptorMap eventData::descriptors;
+
+extern const uint32_t crc32_table[256];
+
+eventData::eventData(const eit_event_struct* e, int size, int type)
+       :ByteSize(size), type(type)
+{
+       if (!e)
+               return;
+       std::list<__u32> d;
+       __u8 *data = (__u8*)e;
+       int ptr=10;
+       int descriptors_length = (data[ptr++]&0x0F) << 8;
+       descriptors_length |= data[ptr++];
+       while ( descriptors_length > 0 )
+       {
+               int descr_len = data[ptr+1] + 2;
+               __u8 *descr = new __u8[descr_len];
+               unsigned int crc=0;
+               int cnt=0;
+               while(cnt < descr_len && descriptors_length > 0)
+               {
+                       crc = (crc << 8) ^ crc32_table[((crc >> 24) ^ data[ptr]) & 0xff];
+                       descr[cnt++] = data[ptr++];
+                       --descriptors_length;
+               }
+               descriptorMap::iterator it =
+                       descriptors.find(crc);
+               if ( it == descriptors.end() )
+               {
+                       CacheSize+=descr_len;
+                       descriptors[crc] = descriptorPair(1, descr);
+               }
+               else
+               {
+                       ++it->second.first;
+                       delete [] descr;
+               }
+               d.push_back(crc);
+       }
+       ByteSize = 12+d.size()*4;
+       EITdata = new __u8[ByteSize];
+       CacheSize+=ByteSize;
+       memcpy(EITdata, (__u8*) e, 12);
+       __u32 *p = (__u32*)(EITdata+12);
+       for (std::list<__u32>::iterator it(d.begin()); it != d.end(); ++it)
+               *p++ = *it;
+}
+
+const eit_event_struct* eventData::get() const
+{
+       static __u8 *data=NULL;
+       if ( data )
+               delete [] data;
+
+       int bytes = 12;
+       std::list<__u8*> d;
+
+// cnt needed bytes
+       int tmp = ByteSize-12;
+       __u32 *p = (__u32*)(EITdata+12);
+       while(tmp>0)
+       {
+               descriptorMap::iterator it =
+                       descriptors.find(*p++);
+               if ( it != descriptors.end() )
+               {
+                       d.push_back(it->second.second);
+                       bytes += it->second.second[1];
+               }
+               bytes += 2; // descr_type, descr_len
+               tmp-=4;
+       }
+
+// copy eit_event header to buffer
+       data = new __u8[bytes];
+       memcpy(data, EITdata, 12);
+
+       tmp=12;
+// copy all descriptors to buffer
+       for(std::list<__u8*>::iterator it(d.begin()); it != d.end(); ++it)
+       {
+               int b=(*it)[1]+2;
+               memcpy(data+tmp, *it, b);
+               tmp+=b;
+       }
+
+       return (eit_event_struct*)data;
+}
+
+eventData::~eventData()
+{
+       if ( ByteSize )
+       {
+               CacheSize-=ByteSize;
+               ByteSize-=12;
+               __u32 *d = (__u32*)(EITdata+12);
+               while(ByteSize)
+               {
+                       descriptorMap::iterator it =
+                               descriptors.find(*d++);
+                       if ( it != descriptors.end() )
+                       {
+                               descriptorPair &p = it->second;
+                               if (!--p.first) // no more used descriptor
+                               {
+                                       CacheSize -= p.second[1]+2;
+                                       delete [] p.second;     // free descriptor memory
+                                       descriptors.erase(it);  // remove entry from descriptor map
+                               }
+                       }
+                       ByteSize-=4;
+               }
+               delete [] EITdata;
+       }
+}
+
+void eventData::load(FILE *f)
+{
+       int size=0;
+       int id=0;
+       __u8 header[2];
+       descriptorPair p;
+       fread(&size, sizeof(int), 1, f);
+       while(size)
+       {
+               fread(&id, sizeof(__u32), 1, f);
+               fread(&p.first, sizeof(int), 1, f);
+               fread(header, 2, 1, f);
+               int bytes = header[1]+2;
+               p.second = new __u8[bytes];
+               p.second[0] = header[0];
+               p.second[1] = header[1];
+               fread(p.second+2, bytes-2, 1, f);
+               descriptors[id]=p;
+               --size;
+               CacheSize+=bytes;
+       }
+}
+
+void eventData::save(FILE *f)
+{
+       int size=descriptors.size();
+       descriptorMap::iterator it(descriptors.begin());
+       fwrite(&size, sizeof(int), 1, f);
+       while(size)
+       {
+               fwrite(&it->first, sizeof(__u32), 1, f);
+               fwrite(&it->second.first, sizeof(int), 1, f);
+               fwrite(it->second.second, it->second.second[1]+2, 1, f);
+               ++it;
+               --size;
+       }
+}
 
 eEPGCache* eEPGCache::instance;
 pthread_mutex_t eEPGCache::cache_lock=
@@ -399,20 +553,14 @@ void eEPGCache::cleanLoop()
        if (!eventDB.empty())
        {
                eDebug("[EPGC] start cleanloop");
-               const eit_event_struct* cur_event;
-               int duration;
 
-// FIXME !!! TIME_CORRECTION
                time_t now = time(0)+eDVBLocalTimeHandler::getInstance()->difference();
 
                for (eventCache::iterator DBIt = eventDB.begin(); DBIt != eventDB.end(); DBIt++)
                {
                        for (timeMap::iterator It = DBIt->second.second.begin(); It != DBIt->second.second.end() && It->first < now;)
                        {
-                               cur_event = (*It->second).get();
-                               duration = fromBCD( cur_event->duration_1)*3600 + fromBCD(cur_event->duration_2)*60 + fromBCD(cur_event->duration_3);
-
-                               if ( now > (It->first+duration) )  // outdated normal entry (nvod references to)
+                               if ( now > (It->first+It->second->getDuration()) )  // outdated normal entry (nvod references to)
                                {
                                        // remove entry from eventMap
                                        eventMap::iterator b(DBIt->second.first.find(It->second->getEventID()));
@@ -449,7 +597,7 @@ eEPGCache::~eEPGCache()
                        delete It->second;
 }
 
-Event *eEPGCache::lookupEvent(const eServiceReferenceDVB &service, int event_id, bool plain)
+RESULT eEPGCache::lookupEvent(const eServiceReferenceDVB &service, int event_id, const eventData *&result )
 {
        singleLock s(cache_lock);
        uniqueEPGKey key( service );
@@ -460,22 +608,19 @@ Event *eEPGCache::lookupEvent(const eServiceReferenceDVB &service, int event_id,
                eventMap::iterator i( It->second.first.find( event_id ));
                if ( i != It->second.first.end() )
                {
-                       if ( service.getServiceType() == 4 ) // nvod ref
-                               return lookupEvent( service, i->second->getStartTime(), plain );
-                       else if ( plain )
-               // get plain data... not in Event Format !!!
-               // before use .. cast it to eit_event_struct*
-                               return (Event*) i->second->get();
-                       else
-                               return new Event( (uint8_t*)i->second->get() /*, (It->first.tsid<<16)|It->first.onid*/ );
+                       result = i->second;
+                       return 0;
                }
                else
+               {
+                       result = 0;
                        eDebug("event %04x not found in epgcache", event_id);
+               }
        }
-       return 0;
+       return -1;
 }
 
-Event *eEPGCache::lookupEvent(const eServiceReferenceDVB &service, time_t t, bool plain )
+RESULT eEPGCache::lookupEvent(const eServiceReferenceDVB &service, time_t t, const eventData *&result )
 // if t == 0 we search the current event...
 {
        singleLock s(cache_lock);
@@ -494,35 +639,26 @@ Event *eEPGCache::lookupEvent(const eServiceReferenceDVB &service, time_t t, boo
                        i--;
                        if ( i != It->second.second.end() )
                        {
-                               const eit_event_struct* eit_event = i->second->get();
-                               int duration = fromBCD(eit_event->duration_1)*3600+fromBCD(eit_event->duration_2)*60+fromBCD(eit_event->duration_3);
-                               if ( t <= i->first+duration )
+                               if ( t <= i->first+i->second->getDuration() )
                                {
-                                       if ( plain )
-                                               // get plain data... not in Event Format !!!
-                                               // before use .. cast it to eit_event_struct*
-                                               return (Event*) i->second->get();
-                                       return new Event( (uint8_t*)i->second->get() /*, (It->first.tsid<<16)|It->first.onid */ );
+                                       result = i->second;
+                                       return 0;
                                }
                        }
                }
 
                for ( eventMap::iterator i( It->second.first.begin() ); i != It->second.first.end(); i++)
                {
-                       const eit_event_struct* eit_event = i->second->get();
-                       int duration = fromBCD(eit_event->duration_1)*3600+fromBCD(eit_event->duration_2)*60+fromBCD(eit_event->duration_3);
-                       time_t begTime = parseDVBtime( eit_event->start_time_1, eit_event->start_time_2,        eit_event->start_time_3, eit_event->start_time_4,       eit_event->start_time_5);
+                       int duration = i->second->getDuration();
+                       time_t begTime = i->second->getStartTime();
                        if ( t >= begTime && t <= begTime+duration) // then we have found
                        {
-                               if ( plain )
-                                       // get plain data... not in Event Format !!!
-                                       // before use .. cast it to eit_event_struct*
-                                       return (Event*) i->second->get();
-                               return new Event( (uint8_t*)i->second->get()/*, (It->first.tsid<<16)|It->first.onid*/ );
+                               result = i->second;
+                               return 0;
                        }
                }
        }
-       return 0;
+       return -1;
 }
 
 void eEPGCache::gotMessage( const Message &msg )
@@ -595,33 +731,43 @@ void eEPGCache::load()
                }
                if ( md5ok )
                {
-                       fread( &size, sizeof(int), 1, f);
-                       while(size--)
+                       char text1[13];
+                       fread( text1, 13, 1, f);
+                       if ( !strncmp( text1, "ENIGMA_EPG_V2", 13) )
                        {
-                               uniqueEPGKey key;
-                               eventMap evMap;
-                               timeMap tmMap;
-                               int size=0;
-                               fread( &key, sizeof(uniqueEPGKey), 1, f);
                                fread( &size, sizeof(int), 1, f);
                                while(size--)
                                {
-                                       int len=0;
-                                       int type=0;
-                                       eventData *event=0;
-                                       fread( &type, sizeof(int), 1, f);
-                                       fread( &len, sizeof(int), 1, f);
-                                       event = new eventData(0, len, type);
-                                       fread( event->EITdata, len, 1, f);
-                                       evMap[ event->getEventID() ]=event;
-                                       tmMap[ event->getStartTime() ]=event;
-                                       ++cnt;
+                                       uniqueEPGKey key;
+                                       eventMap evMap;
+                                       timeMap tmMap;
+                                       int size=0;
+                                       fread( &key, sizeof(uniqueEPGKey), 1, f);
+                                       fread( &size, sizeof(int), 1, f);
+                                       while(size--)
+                                       {
+                                               int len=0;
+                                               int type=0;
+                                               eventData *event=0;
+                                               fread( &type, sizeof(int), 1, f);
+                                               fread( &len, sizeof(int), 1, f);
+                                               event = new eventData(0, len, type);
+                                               event->EITdata = new __u8[len];
+                                               eventData::CacheSize+=len;
+                                               fread( event->EITdata, len, 1, f);
+                                               evMap[ event->getEventID() ]=event;
+                                               tmMap[ event->getStartTime() ]=event;
+                                               ++cnt;
+                                       }
+                                       eventDB[key]=std::pair<eventMap,timeMap>(evMap,tmMap);
                                }
-                               eventDB[key]=std::pair<eventMap,timeMap>(evMap,tmMap);
+                               eventData::load(f);
+                               eDebug("%d events read from /hdd/epg.dat", cnt);
                        }
-                       eDebug("%d events read from /hdd/epg.dat.md5", cnt);
+                       else
+                               eDebug("[EPGC] don't read old epg database");
+                       fclose(f);
                }
-               fclose(f);
        }
 #endif
 }
@@ -653,6 +799,8 @@ void eEPGCache::save()
        int cnt=0;
        if ( f )
        {
+               const char *text = "ENIGMA_EPG_V2";
+               fwrite( text, 13, 1, f );
                int size = eventDB.size();
                fwrite( &size, sizeof(int), 1, f );
                for (eventCache::iterator service_it(eventDB.begin()); service_it != eventDB.end(); ++service_it)
@@ -671,6 +819,7 @@ void eEPGCache::save()
                        }
                }
                eDebug("%d events written to /hdd/epg.dat", cnt);
+               eventData::save(f);
                fclose(f);
                unsigned char md5[16];
                if (!md5_file("/hdd/epg.dat", 1, md5))
@@ -782,7 +931,7 @@ void eEPGCache::channel_data::abortNonAvail()
                        m_ScheduleOtherReader->stop();
                }
                if ( isRunning )
-                       abortTimer.start(50000, true);
+                       abortTimer.start(90000, true);
                else
                {
                        ++state;
@@ -910,6 +1059,7 @@ void eEPGCache::channel_data::readData( const __u8 *data)
                        if ( it == seenSections.end() )
                        {
                                seenSections.insert(sectionNo);
+                               calcedSections.insert(sectionNo);
                                __u32 tmpval = sectionNo & 0xFFFFFF00;
                                __u8 incr = source == NOWNEXT ? 1 : 8;
                                for ( int i = 0; i <= eit->last_section_number; i+=incr )
index 1da612d..8a62325 100644 (file)
@@ -104,45 +104,39 @@ struct hash_uniqueEPGKey
        #define tidMap std::hash_map<__u32, hash_32, equal_32>
 #endif
 
+#define descriptorPair std::pair<int,__u8*>
+#define descriptorMap std::map<__u32, descriptorPair >
+
 class eventData
 {
        friend class eEPGCache;
 private:
        __u8* EITdata;
        int ByteSize;
+       static descriptorMap descriptors;
 public:
        int type;
        static int CacheSize;
-       eventData(const eit_event_struct* e, int size, int type)
-       :ByteSize(size), type(type)
-       {
-               CacheSize+=size;
-               EITdata = new __u8[size];
-               if (e)
-                       memcpy(EITdata, (__u8*) e, size);
-       }
-       ~eventData()
-       {
-               CacheSize-=ByteSize;
-               delete [] EITdata;
-       }
+       static void load(FILE *);
+       static void save(FILE *);
+       eventData(const eit_event_struct* e, int size, int type);
+       ~eventData();
+       const eit_event_struct* get() const;
        operator const eit_event_struct*() const
        {
-               return (const eit_event_struct*) EITdata;
-       }
-       const eit_event_struct* get() const
-       {
-               return (const eit_event_struct*) EITdata;
+               return get();
        }
        int getEventID()
        {
-               return HILO( ((const eit_event_struct*) EITdata)->event_id );
+               return (EITdata[0] << 8) | EITdata[1];
        }
        time_t getStartTime()
        {
-               return parseDVBtime(
-                       EITdata[2], EITdata[3],
-                       EITdata[4], EITdata[5], EITdata[6]);
+               return parseDVBtime(EITdata[2], EITdata[3], EITdata[4], EITdata[5], EITdata[6]);
+       }
+       int getDuration()
+       {
+               return fromBCD(EITdata[7])*3600+fromBCD(EITdata[8])*60+fromBCD(EITdata[9]);
        }
 };
 
@@ -239,30 +233,64 @@ public:
        // called from main thread
        inline void Lock();
        inline void Unlock();
-       Event *lookupEvent(const eServiceReferenceDVB &service, int event_id, bool plain=false );
-       Event *lookupEvent(const eServiceReferenceDVB &service, time_t=0, bool plain=false );
-       const eventMap* getEventMap(const eServiceReferenceDVB &service);
-       const timeMap* getTimeMap(const eServiceReferenceDVB &service);
+
+       // result Event * must be deleted by caller of lookupEvent
+       inline RESULT lookupEvent(const eServiceReferenceDVB &service, int event_id, Event *& );
+       inline RESULT lookupEvent(const eServiceReferenceDVB &service, time_t , Event *& );
+
+       // methods for faster use.. but not thread save ..
+       // Lock and Unlock should be used !!
+       RESULT lookupEvent(const eServiceReferenceDVB &service, int event_id, const eventData *& );
+       RESULT lookupEvent(const eServiceReferenceDVB &service, time_t , const eventData *& );
+
+       inline RESULT getEventMap(const eServiceReferenceDVB &service, const eventMap *& );
+       inline RESULT getTimeMap(const eServiceReferenceDVB &service, const timeMap *& );
 };
 
 TEMPLATE_TYPEDEF(ePtr<eEPGCache>,eEPGCachePtr);
 
-inline const eventMap* eEPGCache::getEventMap(const eServiceReferenceDVB &service)
+inline RESULT eEPGCache::lookupEvent(const eServiceReferenceDVB &service, time_t t, Event *& result )
+{
+       const eventData *data=0;
+       RESULT ret = lookupEvent(service, t, data);
+       if ( !ret && data )
+               result = new Event((uint8_t*)data->get());
+       return ret;
+}
+
+inline RESULT eEPGCache::lookupEvent(const eServiceReferenceDVB &service, int event_id, Event *& result)
+{
+       const eventData *data=0;
+       RESULT ret = lookupEvent(service, event_id, data);
+       if ( !ret && data )
+               result = new Event((uint8_t*)data->get());
+       return ret;
+}
+
+inline RESULT eEPGCache::getEventMap(const eServiceReferenceDVB &service, const eventMap *& result)
 {
        eventCache::iterator It = eventDB.find( service );
        if ( It != eventDB.end() && It->second.first.size() )
-               return &(It->second.first);
-       else
+       {
+               result=&(It->second.first);
                return 0;
+       }
+       else
+               result=0;
+       return -1;
 }
 
-inline const timeMap* eEPGCache::getTimeMap(const eServiceReferenceDVB &service)
+inline RESULT eEPGCache::getTimeMap(const eServiceReferenceDVB &service, const timeMap *& result)
 {
        eventCache::iterator It = eventDB.find( service );
        if ( It != eventDB.end() && It->second.second.size() )
-               return &(It->second.second);
-       else
+       {
+               result=&(It->second.second);
                return 0;
+       }
+       else
+               result=0;
+       return -1;
 }
 
 inline void eEPGCache::Lock()