Refactor Jackett parser script, fix errors, add sorting
Tracker User tracker-user@users.noreply.github.com
Wed, 07 Apr 2021 04:00:46 +0300
1 files changed,
127 insertions(+),
145 deletions(-)
jump to
M
read.js
→
read.js
@@ -1,149 +1,113 @@
-// list of trackers to ignore from Jackett -const jackettIgnoreList = [ - 'AlphaRatio', - 'AnimeBytes', - 'AnimeTorrents', - 'Anthelion', - 'AsianCinema', - 'Avistaz', - 'Awesome-HD', - 'BakaBT', - 'Beyond-HD', - 'Beyond-HD (OneURL)', - 'Blutopia', - 'BroadcastTheNet', - 'BrokenStones', - 'CGPeers', - 'DesiTorrents', - 'DXDHD', - 'Empornium', - 'ExoticaZ', - 'Femdomcult', - 'GazelleGames', - 'HD-Space', - 'HDBits', - 'HDBits (API)', - 'HDChina', - 'IPTorrents', - 'JPopsuki', - 'LegacyHD', - 'MoreThanTV', - 'MyAnonamouse', - 'MySpleen', - 'Nebulance', - 'notwhat.cd', - 'Orpheus', - 'PassThePopcorn', - 'PirateTheNet', - 'PixelCove', - 'Pornbay', - 'PrivateHD', - 'Psytorrents', - 'PTFiles', - 'Racing4Everyone (R4E)', - 'RacingForMe', - 'Redacted', - 'SceneTime', - 'Secret Cinema', - 'ShareUniversity', - 'SportsCult', - 'Superbits', - 'The Geeks', - 'The Place', - 'TorrentBD', - 'TorrentLeech', - 'TV-Vault', - 'XWtorrents' -] - const path = require('path'); const fs = require('fs'); const yaml = require('js-yaml'); -// joining path of directory +// joining path of directory const directoryPath = path.join(__dirname, 'Jackett/src/Jackett.Common/Definitions'); // passsing directoryPath and callback function let trackers = {}; trackers.trackers = [] -const categoriesPath = path.join(__dirname, 'Jackett/src/Jackett.Common/Models/TorznabCatType.generated.cs'); +// list of trackers to ignore from Jackett +const mainJson = JSON.parse(fs.readFileSync('trackers.json')); +const mainIgnore = []; + +mainJson.trackers.forEach(function(obj) { + mainIgnore.push(obj.Name.toLowerCase()); +}); + +const customIgnore = [ + 'beyond-hd (oneurl)', + 'hdbits (api)', + 'racing4everyone (r4e)', + 'the geeks', + 'the place', + 'xwtorrents' +] + +const jackettIgnoreList = mainIgnore.concat(customIgnore); + +/*const categoriesPath = path.join(__dirname, 'Jackett/src/Jackett.Common/Models/TorznabCatType.generated.cs'); try { const fileContents = fs.readFileSync(path.join(categoriesPath, file), 'utf8'); // SubCategories } catch (err) { - // -} + // +}*/ // clean up duplicate types // see Jackett/src/Jackett.Common/Models/TorznabCatType.generated.cs -function cleanType(type) { - let types = new Map([ - ['AudioAudiobook', 'Audiobooks'], - ['AudioForeign', 'Audio'], - ['AudioLossless', 'Audio'], - ['AudioMP3', 'Audio'], - ['AudioOther', 'Audio'], - ['AudioVideo', 'Audio'], - ['BooksComics', 'Comics'], - ['BooksEbook', 'eBooks'], - ['BooksForeign', 'Books'] - ['BooksMagazines', 'Magazines'], - ['BooksOther', 'Books'], - ['BooksTechnical', 'Technical Books'], - ['Console3DS', 'Console'], - ['ConsoleNDS', 'Console'], - ['ConsoleOther', 'Console'], - ['ConsolePS3', 'Console'], - ['ConsolePS4', 'Console'], - ['ConsolePSP', 'Console'], - ['ConsolePSVita', 'Console'], - ['ConsoleWii', 'Console'], - ['ConsoleWiiU', 'Console'], - ['ConsoleWiiwareVC', 'Console'], - ['ConsoleXbox', 'Console'], - ['ConsoleXbox360', 'Console'], - ['ConsoleXBOX360DLC', 'Console'], - ['ConsoleXboxOne', 'Console'], - ['Movies3D', 'Movies'], - ['MoviesBluRay', 'Movies'], - ['MoviesDVD', 'Movies'], - ['MoviesForeign', 'Movies'], - ['MoviesHD', 'Movies'], - ['MoviesOther', 'Movies'], - ['MoviesSD', 'Movies'], - ['MoviesUHD', 'Movies'], - ['MoviesWEBDL', 'Movies'], - ['Other', 'General'], - ['OtherHashed', 'General'], - ['OtherMisc', 'General'], - ['PC0day', 'PC'], - ['PCGames', 'Games'], - ['PCISO', 'PC'], - ['PCMac', 'Mac Software'], - ['PCPhoneAndroid', 'Android'], - ['PCPhoneIOS', 'iOS'], - ['PCPhoneOther', 'Phone'], - ['TVAnime', 'Anime'], - ['TVDocumentary', 'TV'], - ['TVFOREIGN', 'TV'], - ['TVHD', 'TV'], - ['TVOTHER', 'TV'], - ['TVSD', 'TV'], - ['TVSport', 'Sports'], - ['TVUHD', 'TV'], - ['TVWEBDL', 'TV'], - ['XXXDVD', 'XXX'], - ['XXXImageset', 'XXX'], - ['XXXOther', 'XXX'], - ['XXXPacks', 'XXX'], - ['XXXWMV', 'XXX'], - ['XXXx264', 'XXX'], - ['XXXXviD', 'XXX'], - ]) - - return types.get(type) || type +const cleanTypes = { + "audioaudiobook": "Audiobooks", + "audioforeign": "Audio", + "audiolossless": "Audio", + "audiomp3": "Audio", + "audioother": "Audio", + "audiovideo": "Audio", + "bookscomics": "Comics", + "booksebook": "Books", + "booksforeign": "Books", + "booksmagazines": "Magazines", + "booksmags": "Magazines", + "booksother": "Books", + "bookstechnical": "Books", + "console3ds": "Console", + "consolends": "Console", + "consoleother": "Console", + "consoleps3": "Console", + "consoleps4": "Console", + "consolepsp": "Console", + "consolepsvita": "Console", + "consolewii": "Console", + "consolewiiu": "Console", + "consolewiiwarevc": "Console", + "consolexbox": "Console", + "consolexbox360": "Console", + "consolexbox360dlc": "Console", + "consolexboxone": "Console", + "movies3d": "Movies", + "moviesbluray": "Movies", + "moviesdvd": "Movies", + "moviesforeign": "Movies", + "movieshd": "Movies", + "moviesother": "Movies", + "moviessd": "Movies", + "moviesuhd": "Movies", + "movieswebdl": "Movies", + "other": "General", + "otherhashed": "General", + "othermisc": "General", + "pc0day": "PC", + "pcgames": "Games", + "pciso": "PC", + "pcmac": "Mac Software", + "pcmobileandroid": "Android", + "pcmobileios": "iOS", + "pcmobileother": "Phone", + "pcphoneandroid": "Android", + "pcphoneios": "iOS", + "pcphoneother": "Phone", + "tvanime": "Anime", + "tvdocumentary": "TV", + "tvforeign": "TV", + "tvhd": "TV", + "tvother": "TV", + "tvsd": "TV", + "tvsport": "Sports", + "tvuhd": "TV", + "tvwebdl": "TV", + "xxxdvd": "XXX", + "xxximageset": "XXX", + "xxxother": "XXX", + "xxxpack": "XXX", + "xxxpacks": "XXX", + "xxxsd": "XXX", + "xxxuhd": "XXX", + "xxxwmv": "XXX", + "xxxx264": "XXX", + "xxxxvid": "XXX", } function cleanTypeDefinition(type) {@@ -158,15 +122,16 @@ fs.readdir(directoryPath, function (err, files) {
// handling error if (err) { return console.log('Unable to scan directory: ' + err); - } + } // listing all files using forEach files.forEach(function (file) { // Do whatever you want to do with the file try { let fileContents = fs.readFileSync(path.join(directoryPath, file), 'utf8'); - let data = yaml.safeLoad(fileContents, {skipInvalid: true}); + // "json: true" helps with duplicated keys + let data = yaml.safeLoad(fileContents, {skipInvalid: true, json: true}); - if (data.type == 'private') { + if (data.type === 'private') { let tracker = { name: '', description: '',@@ -175,7 +140,7 @@ }
tracker.name = data.name tracker.description = data.description let type = [] - + if (data.caps && data.caps.categorymappings) { data.caps.categorymappings.forEach(function(category) { type.push(cleanTypeDefinition(category.cat.split("/")[0]))@@ -183,7 +148,7 @@ })
// remove duplicates type = Array.from(new Set(type)) tracker.type = type.join(", ") - } else if (data.caps.categories) { + } else if (data.caps.categories && data.caps.categorymappings) { data.caps.categorymappings.forEach(function(category) { //console.log(category) type.push(cleanTypeDefinition(category.cat.split("/")[0]))@@ -199,8 +164,11 @@ trackers.trackers.push(tracker)
} } catch (err) { // YAMLException + console.log(err); } }); + + console.log('Without indexers: ' + trackers.trackers.length); const indexersPath = path.join(__dirname, 'Jackett/src/Jackett.Common/Indexers');@@ -208,38 +176,42 @@ fs.readdir(indexersPath, function (err, files) {
// handling error if (err) { return console.log('Unable to scan directory: ' + err); - } + } // listing all files using forEach files.forEach(function (file) { + // Skip directories + if (!file.endsWith('.cs')) { + return; + } // Do whatever you want to do with the file try { const fileContents = fs.readFileSync(path.join(indexersPath, file), 'utf8'); - - const isPrivate = fileContents.match(/Type.*"private"/i) + const typePrivate = /Type\s+=\s+"private"/i + const typeAvistaz = /AvistazTracker/i + const isPrivate = typePrivate.test(fileContents) || typeAvistaz.test(fileContents) if (isPrivate) { - let tracker = { name: '', description: '', type: '' } - const res = fileContents.match( /:\sbase\([^{]*"/s ) + const res = fileContents.match(/:\s+base\([^{]*"/s) const lines = res[0].match(/[^\r\n]+/g); for (const line of lines) { - const name = line.match(/.*name\:\s\"([\w\s\.\-\()]+)\"/i) + const name = line.match(/.*name:\s+"([\w\s.\-()]+)"/i) if (name) { tracker.name = name[1] } else { - let name2 = line.match(/.*base\(\"([\w\s\.\-\()]+)\"/i) + let name2 = line.match(/.*base\("([\w\s.\-()]+)"/i) if (name2) { tracker.name = name2[1] } } - const description = line.match(/.*[desc|description]\:\s\"([^\"]+)\"/i) + const description = line.match(/.*(?:desc|description):\s+"([^"]+)"/i) if (description) { tracker.description = description[1] }@@ -248,10 +220,15 @@
let type = [] const fileLines = fileContents.match(/[^\r\n]+/g) for (const line of fileLines) { - if (line.includes("AddCategoryMapping")) { + if (line && line.includes("AddCategoryMapping")) { const category = line.match(/TorznabCatType\.([^,]+),/i) if (category) { - type.push(cleanType(category[1])) + const toReplace = cleanTypes.hasOwnProperty(category[1].toLowerCase()) + if (toReplace) { + type.push(cleanTypes[category[1].toLowerCase()]) + } else { + type.push(category[1]) + } } } }@@ -262,11 +239,16 @@
trackers.trackers.push(tracker) } } catch (err) { - // + console.log(err); } }); - trackers.trackers = trackers.trackers.filter((t) => !jackettIgnoreList.includes(t.name)) + console.log('With indexers: ' + trackers.trackers.length); + + trackers.trackers = trackers.trackers.filter((t) => !jackettIgnoreList.includes(t.name.toLowerCase())) + trackers.trackers.sort(function(a, b) { + return a.name.localeCompare(b.name) + }); let data = JSON.stringify(trackers); fs.writeFileSync('trackers2.json', data);