""" Code written by Jason Moiron. Permission to use/distribute/modify under the GPL v2: http://www.gnu.org/licenses/gpl.txt All functions work on fileobjects and assume that the seek position in the file was set to "0". If you do not have the file open, pass open(file, 'rb') to these functions. If you want the exporting to a string, check StringIO. Extending: The DAT formats all provide the same members. Some of this is done in the super classes, some must be provided in methods by subclasses. Reference the clrMamePro and romcenter implementations (although there are hints in the supers): DatFile: header : python dictionary of header attrs keys: ['author', 'email', 'homepage', 'url', 'version', 'comment', 'name', 'desc', 'category'] games : python list of Game objects Game: meta : python dictionary of meta attrs keys: ['name', 'desc', 'parent', 'parent_desc', 'number', 'region_code', 'region', 'group'] rom : python dictionary of meta attrs keys: ['crc', 'name', 'size', 'romof', 'merge'] additionally, all 'meta' keys and rom['crc'] have aliases in Game, ex. Game.crc or Game.number """ #TODO: unit testing import sys,re import ndscodes, romlib def rom_cf(x,y): if x.number > y.number: return 1 elif x.number == y.number: #printv('Note: duplicate rom found for %s ("%s" and "%s")' % (x.number, x.filename, y.filename)) x.dupe = True y.dupe = True return 0 else: return -1 phr_hwo_list = [(799, 'NinjaPass X9TF'), (463, 'MAX Media Launcher')] # make sure that the phr_hwo_list is in decending order... def _hws(x,y): if x[0] > y[0]: return -1 elif x[0] == y[0]: return 0 else: return 1 phr_hwo_list.sort(_hws) class DatFileException(Exception): pass def openDat(fileobj_or_name): """Auto-creates a DatFile from a fileobject or name.""" if isinstance(fileobj_or_name, file): return autoIdentify(fileobj_or_name) elif isinstance(fileobj_or_name, str) or \ isinstance(fileobj_or_name, unicode): return autoIdentify(open(fileobj_or_name)) else: raise DatFileException, "Expecting string or file object" def autoIdentify(fileobj): """Auto-creates a DatFile from a file object.""" if is_clrmamepro(fileobj): return clrDatFile(fileobj) elif is_romcenter(fileobj): return rcDatFile(fileobj) raise DatFileException, "Could not determine the dat format" def is_clrmamepro(fileobj): """Checks if a fileobject is a clrMamePro dat file.""" magic = fileobj.read(10) fileobj.seek(0) if magic == 'clrmamepro': return True return False def is_romcenter(fileobj): """Checks if a fileobject is a RomCenter dat file.""" magic = fileobj.read(9) fileobj.seek(0) if magic == '[CREDITS]': return True return False class Game: """Parent class for Game objects. Instantiating will throw an Exception. Game represents a record of a rom inside a DatFile object. When extending, please implement _at least_ 'parse_meta' and 'parse_rom'. The expected format of meta and rom are dictionaries containing: meta: {'name':, 'desc':, 'parent':, 'parent_desc':,} rom: {'name':, 'crc':, 'size':, 'romof':, 'merge':} Child classes should also implement '_format' as a static method. There's no magic now to auto-format on the export, so add relevant lines to Game's 'export' method. """ def __init__(self, lines): self._lines = lines self.meta = self.parse_meta() self.rom = self.parse_rom() self.meta['number'] = self.parse_num() # this region tango ensures that the region_code is 1 letter self.meta['orig_region_code'] = self.parse_region() try: self.meta['region'] = ndscodes.region_map[self.meta['orig_region_code']] except: print self.meta['orig_region_code'] print self._lines self.meta['region_code'] = ndscodes.map_region(self.meta['region']) self.meta['group'] = self.parse_group() # lets play namespace poluting game for key,val in self.meta.items(): setattr(self, key, val) self.crc = self.rom['crc'] self.clean_name = self.parse_clean_name() # implement these using these types def parse_meta(self): return {} def parse_rom(self): return {} def parse_num(self): """Parse the number out of a meta/file name.""" return romlib.parse_num(self.meta['name']) def parse_region(self): """Parse the region out of a meta/file name.""" return romlib.parse_region(self.meta['name']) def parse_group(self): """Parse the release group out of a meta/file name.""" return romlib.parse_group(self.meta['name']) def parse_clean_name(self): return romlib.parse_clean_name(self.meta['name'], number=self.meta['number'], group=self.meta['group'], region=self.meta['orig_region_code']) def update_rom_number(self, number): """Change the number of this rom. In addition to the number, any meta and rom information that contains the number must be updated as well.""" oldnum = self.meta['number'] # change the numbers first... self.meta['number'] = number self.number = number # meta['name'], meta['desc'], rom['name'], rom['romof'] def crn(val): newval = val.replace('%04d' % oldnum, '%04d' % number) if newval == val: newval = val.replace(str(oldnum), str(number)) return newval for item in ('name', 'desc', 'parent', 'parent_desc'): self.meta[item] = crn(self.meta[item]) setattr(self, item, crn(getattr(self, item))) self.rom['name'] = crn(self.rom['name']) self.rom['romof'] = crn(self.rom['romof']) def __repr__(self): _r = lambda x: self.rom[x] s = '' % [(_r('name'), _r('crc'))] return s def export(self, fileobj=sys.stdout, format=False): if not format: format = self.__class__.__name__ if format.startswith('clr'): s = clrGame._format(self) elif format.startswith('rc'): s = rcGame._format(self) else: s = repr(self) fileobj.write(s) class rcGame(Game): def parse_meta(self): meta = {} spl = self._lines.split('\xac') meta['parent'] = spl[1] meta['parent_desc'] = spl[2] meta['name'] = spl[3] meta['desc'] = spl[4] return meta def parse_rom(self): rom = {} spl = self._lines.split('\xac') rom['name'] = spl[5] rom['crc'] = spl[6] rom['size'] = spl[7] rom['romof'] = spl[8] rom['merge'] = spl[9] return rom @staticmethod def _format(self): sep = '\xac' _ = lambda x: self.meta[x] _r = lambda x: self.rom[x] s = sep s += sep.join([_('parent'), _('parent_desc'), _('name'), _('desc'), _r('name'), _r('crc'), _r('size'), _r('romof'), _r('merge')]) s += sep + '\n' return s class clrGame(Game): def parse_meta(self): meta = {} for line in self._lines: if line.startswith('name'): meta['name'] = line.split('"')[1] elif line.startswith('description'): meta['desc'] = line.split('"')[1] elif line.startswith('cloneof'): meta['parent'] = line.split('"')[1] if 'parent' not in meta.keys(): meta['parent'] = '' meta['parent_desc'] = '' return meta def parse_rom(self): rom_re = re.compile('.*?"(?P.*?)".*?crc (?P[0-9a-fA-F]{8})') size_re = re.compile('size (?P[0-9]*)') for line in self._lines: if line.startswith('rom'): m1 = rom_re.search(line) m2 = size_re.search(line) d = {'size' : '', 'romof': '', 'merge': '',} d.update({'name' : m1.group('name'), 'crc' : m1.group('crc'),}) if m2: d.update({'size' : m2.group('size')}) return d @staticmethod def _format(self): if isinstance(self.number, int): numstr = '%04d' % self.number elif isinstance(self.number, str): numstr = self.number else: raise Exception, 'Number is not int or string: %s' % (self.number) s = 'game (\n\t' s += 'name "%s"\n\t' % self.name s += 'description "%s"\n\t' % self.desc s += 'rom ( name "%s" ' % self.rom['name'] if self.rom['size']: s += 'size %s ' % self.rom['size'] s += 'crc %s )\n' % self.rom['crc'] s += ')\n\n' return s class DatFile: def __init__(self, dat): self._lines = [l.strip() for l in dat] self._headerlines = self._get_headerlines() self._gamelines = self._get_gamelines() # creates 'header' self.header = self.parse_header() # creates [games, ...] self.games = self.parse_games() self.sanity_check() def sanity_check(self): # make sure that games line up with their numbers for idx,game in enumerate(self.games): if game.number != idx + 1: print "Error: Index (%s) contains rom (%s). (hole in list for game %04d)" % (idx + 1, game.number, idx+1) sys.exit(1) # when you implement these, make sure you return these types def parse_games(self): return [] def parse_header(self): return {} def remove_game(self, game_number): """ Remove the game with the game_number. This is far more complex than it seems, since Game objects sometimes have release numbers (in NDS and GBA files) that are littered all over Filename and Romname attributes. """ #FIXME: remove_game is not safe against string numbers self.sanity_check() del self.games[game_number - 1] for game in self.games: if game.number > game_number: game.update_rom_number(game.number - 1) self.sanity_check() def add_game(self, gameobj): """ Add a game to the list. This is also more complex than it seems, especially when gameobj.number > games[-1].number """ #TODO: Write this function.. if it is necessary! pass def __repr__(self): return 'DatFile %s v%s [games : %d]' % (self.header['author'], self.header['version'], len(self.games)) # this should be used to re-write changed DATs def export(self, fileobj=sys.stdout, format=False): if not format: format = self.__class__.__name__ if format.startswith('clr'): s = clrDatFile._format(self) elif format.startswith('rc'): s = rcDatFile._format(self) else: s = repr(self) fileobj.write(s) for game in self.games: game.export(fileobj, format) def findgame(self, name): matchlist = [] for game in self.games: if game.name.rfind(name) > -1: matchlist.append(game) return matchlist class rcDatFile(DatFile): dftype = 'RomCenter' def _get_headerlines(self): l = [] for line in self._lines: if line != '[GAMES]': l.append(line) else: break return l def _get_gamelines(self): return self._lines[len(self._headerlines)+1:] def parse_header(self): h = {} credits = [] for line in self._headerlines: if line.upper() != '[DAT]': credits.append(line) else: break _ = lambda x: '='.join(x.split('=')[1:]) starts = lambda x,y: x.lower().startswith(y) for line in credits: if starts(line, 'author'): h['author'] = _(line) elif starts(line, 'email'): h['email'] = _(line) elif starts(line, 'homepage'): h['homepage'] = _(line) elif starts(line, 'url'): h['url'] = _(line) elif starts(line, 'version'): h['version'] = _(line) elif starts(line, 'comment'): h['comment'] = _(line) h['name'] = h['comment'] h['desc'] = h['comment'] h['category'] = '' return h def parse_games(self): g = [] for line in self._gamelines: g.append(rcGame(line)) g.sort(rom_cf) # strip out nukes from ADVANScENe dats g = [game for game in g if game.number != 'xxxx'] return g @staticmethod def _format(self): s = '[CREDITS]\n' s += 'Author=%s\n' % self.header['author'] s += 'Email=%s\n' % self.header['email'] s += 'Homepage=%s\n' % self.header['homepage'] s += 'Url=%s\n' % self.header['url'] s += 'Version=%s\n' % self.header['version'] s += 'Comment=%s\n' % self.header['comment'] # NOTE: pocketheaven lists actually have v2.50 features (url, etc) s += '[DAT]\n' s += 'version=2.50\n' # NOTE: not sure what this section does... s += '[EMULATOR]\n' s += 'refname=iDeaS\n' s += '[GAMES]\n' return s class clrDatFile(DatFile): dftype = 'ClrMamePro' def _get_headerlines(self): l = [] for line in self._lines: if line: l.append(line) else: break return l def _get_gamelines(self): return self._lines[len(self._headerlines)+1:] def parse_header(self): h = {} for line in self._headerlines: if line.startswith('name'): nre = re.compile(r'^name "?(?P.*?)"?$') h['name'] = nre.match(line).group('name') elif line.startswith('description'): dre = re.compile(r'^description "?(?P.*?)"?$') h['desc'] = dre.match(line).group('desc') elif line.startswith('category'): ore = re.compile(r'^category "?(?P.*?)"?$') h['category'] = ore.match(line).group('cat') elif line.startswith('version'): vre = re.compile(r'^version \(?(?P[\d\w_-]+)\)?') h['version'] = vre.search(line).group('version') elif line.startswith('author'): h['author'] = ' '.join(line.split()[1:]) h['comment'] = h['desc'] h['email'] = '' h['homepage'] = '' h['url'] = '' return h def parse_games(self): def chunklines(lines): chunk = [] for line in lines: if line: chunk.append(line) else: yield chunk chunk = [] g = [] for chunk in chunklines(self._gamelines): g.append(clrGame(chunk)) # get these in order of rom number g.sort(rom_cf) # strip out nukes from ADVANScENe dats g = [game for game in g if game.number != 'xxxx'] return g @staticmethod def format(self): s = "clrmamepro (\n\t" s += 'name "%s"\n\t' % self.header['name'] s += 'description "%s"\n\t' % self.header['desc'] s += 'category "%s"\n\t' % self.header['category'] s += 'version %s\n\t' % self.header['version'] s += 'author %s\n' % self.header['author'] s += ')\n\n' return s # some naive testing if __name__ == '__main__': cf1 = open('test/dat/ADVANsCEne_NDS_Release_Dat_859.dat') cf2 = open('test/dat/PocketHeaven_GBA_Release_List_Roms(2643)[CM].dat') try: c1df = openDat(cf1) c2df = openDat(cf2) r1df = openDat('test/dat/PocketHeaven_NDS-Numbered_Release_List_Roms(0821)[RC].dat') if not isinstance(r1df, rcDatFile): print 'File %s was auto-identified incorrectly. (should have been "romcenter")' % r1df.name if not isinstance(c1df, clrDatFile): print 'File %s was auto-identified incorrectly. (should have been "clrmamepro")' % cf1.name if not isinstance(c2df, clrDatFile): print 'File %s was auto-identified incorrectly. (should have been "clrmamepro")' % cf2.name if len(c1df.games) != 822: print 'Lost games in %s (should be 822)' % cf1.name if len(c2df.games) != 2642: print 'Lost games in %s (should be 2642)' % cf2.name if len(r1df.games) != 821: print 'Lost games in %s (should be 821)' % r1df.name except: print 'An error occured during testing:' import traceback traceback.print_exc()