diff --git a/.prettierrc b/.prettierrc index 281ed98..3099fbc 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,7 +1,6 @@ { - "tabWidth": 2, + "tabWidth": 4, "useTabs": false, "semi": false, - "singleQuote": true, - "objectWrap": "collapse" + "singleQuote": true } diff --git a/deno.lock b/deno.lock index f780d3d..79af15c 100644 --- a/deno.lock +++ b/deno.lock @@ -31,6 +31,7 @@ } }, "redirects": { + "https://deno.land/x/terminal_size/mod.ts": "https://deno.land/x/terminal_size@0.1.0/mod.ts", "https://esm.sh/@types/boolbase@~1.0.3/index.d.ts": "https://esm.sh/@types/boolbase@1.0.3/index.d.ts", "https://esm.sh/@types/safer-buffer@~2.1.3/index.d.ts": "https://esm.sh/@types/safer-buffer@2.1.3/index.d.ts", "https://esm.sh/boolbase@^1.0.0?target=deno": "https://esm.sh/boolbase@1.0.0?target=deno", @@ -89,6 +90,18 @@ "https://esm.sh/whatwg-mimetype@^4.0.0?target=denonext": "https://esm.sh/whatwg-mimetype@4.0.0?target=denonext" }, "remote": { + "https://deno.land/std@0.122.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", + "https://deno.land/std@0.122.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", + "https://deno.land/std@0.122.0/fmt/colors.ts": "8368ddf2d48dfe413ffd04cdbb7ae6a1009cf0dccc9c7ff1d76259d9c61a0621", + "https://deno.land/std@0.122.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", + "https://deno.land/std@0.122.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", + "https://deno.land/std@0.122.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", + "https://deno.land/std@0.122.0/path/common.ts": "f41a38a0719a1e85aa11c6ba3bea5e37c15dd009d705bd8873f94c833568cbc4", + "https://deno.land/std@0.122.0/path/glob.ts": "7bf2349e818e332a830f3d8874c3f45dd7580b6c742ed50dbf6282d84ab18405", + "https://deno.land/std@0.122.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", + "https://deno.land/std@0.122.0/path/posix.ts": "34349174b9cd121625a2810837a82dd8b986bbaaad5ade690d1de75bbb4555b2", + "https://deno.land/std@0.122.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", + "https://deno.land/std@0.122.0/path/win32.ts": "11549e8c6df8307a8efcfa47ad7b2a75da743eac7d4c89c9723a944661c8bd2e", "https://deno.land/std@0.177.1/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", "https://deno.land/std@0.177.1/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", "https://deno.land/std@0.177.1/async/abortable.ts": "73acfb3ed7261ce0d930dbe89e43db8d34e017b063cf0eaa7d215477bf53442e", @@ -377,6 +390,37 @@ "https://deno.land/std@0.177.1/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", "https://deno.land/std@0.177.1/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", "https://deno.land/std@0.177.1/testing/asserts.ts": "984ab0bfb3faeed92ffaa3a6b06536c66811185328c5dd146257c702c41b01ab", + "https://deno.land/std@0.97.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", + "https://deno.land/std@0.97.0/_util/os.ts": "e282950a0eaa96760c0cf11e7463e66babd15ec9157d4c9ed49cc0925686f6a7", + "https://deno.land/std@0.97.0/encoding/base64.ts": "eecae390f1f1d1cae6f6c6d732ede5276bf4b9cd29b1d281678c054dc5cc009e", + "https://deno.land/std@0.97.0/encoding/hex.ts": "f952e0727bddb3b2fd2e6889d104eacbd62e92091f540ebd6459317a61932d9b", + "https://deno.land/std@0.97.0/fs/_util.ts": "f2ce811350236ea8c28450ed822a5f42a0892316515b1cd61321dec13569c56b", + "https://deno.land/std@0.97.0/fs/ensure_dir.ts": "b7c103dc41a3d1dbbb522bf183c519c37065fdc234831a4a0f7d671b1ed5fea7", + "https://deno.land/std@0.97.0/fs/exists.ts": "b0d2e31654819cc2a8d37df45d6b14686c0cc1d802e9ff09e902a63e98b85a00", + "https://deno.land/std@0.97.0/hash/_wasm/hash.ts": "cb6ad1ab429f8ac9d6eae48f3286e08236d662e1a2e5cfd681ba1c0f17375895", + "https://deno.land/std@0.97.0/hash/_wasm/wasm.js": "94b1b997ae6fb4e6d2156bcea8f79cfcd1e512a91252b08800a92071e5e84e1a", + "https://deno.land/std@0.97.0/hash/hasher.ts": "57a9ec05dd48a9eceed319ac53463d9873490feea3832d58679df6eec51c176b", + "https://deno.land/std@0.97.0/hash/mod.ts": "5d032bd34186cda2f8d17fc122d621430953a6030d4b3f11172004715e3e2441", + "https://deno.land/std@0.97.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", + "https://deno.land/std@0.97.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", + "https://deno.land/std@0.97.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", + "https://deno.land/std@0.97.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a", + "https://deno.land/std@0.97.0/path/glob.ts": "314ad9ff263b895795208cdd4d5e35a44618ca3c6dd155e226fb15d065008652", + "https://deno.land/std@0.97.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", + "https://deno.land/std@0.97.0/path/posix.ts": "f56c3c99feb47f30a40ce9d252ef6f00296fa7c0fcb6dd81211bdb3b8b99ca3b", + "https://deno.land/std@0.97.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", + "https://deno.land/std@0.97.0/path/win32.ts": "77f7b3604e0de40f3a7c698e8a79e7f601dc187035a1c21cb1e596666ce112f8", + "https://deno.land/x/cache@0.2.13/cache.ts": "4005aad54fb9aac9ff02526ffa798032e57f2d7966905fdeb7949263b1c95f2f", + "https://deno.land/x/cache@0.2.13/deps.ts": "6f14e76a1a09f329e3f3830c6e72bd10b53a89a75769d5ea886e5d8603e503e6", + "https://deno.land/x/cache@0.2.13/directories.ts": "ef48531cab3f827252e248596d15cede0de179a2fb15392ae24cf8034519994f", + "https://deno.land/x/cache@0.2.13/file.ts": "5abe7d80c6ac594c98e66eb4262962139f48cd9c49dbe2a77e9608760508a09a", + "https://deno.land/x/cache@0.2.13/file_fetcher.ts": "5c793cc83a5b9377679ec313b2a2321e51bf7ed15380fa82d387f1cdef3b924f", + "https://deno.land/x/cache@0.2.13/helpers.ts": "d1545d6432277b7a0b5ea254d1c51d572b6452a8eadd9faa7ad9c5586a1725c4", + "https://deno.land/x/cache@0.2.13/mod.ts": "3188250d3a013ef6c9eb060e5284cf729083af7944a29e60bb3d8597dd20ebcd", + "https://deno.land/x/plug@0.5.1/deps.ts": "0f53866f60dc4f89bbc3be9d096f7501f2068a51923db220f43f0ad284b6b892", + "https://deno.land/x/plug@0.5.1/plug.ts": "620f9454df049228ba715b215582200a344f33f37605d5b4dfce5bf89c7e9815", + "https://deno.land/x/terminal_size@0.1.0/bindings/bindings.ts": "712ba91bfc7c93d5bbad2e6c996ccc062d9a12ac3c0fa258ed068f666ce43d3b", + "https://deno.land/x/terminal_size@0.1.0/mod.ts": "6ab2530394db2675f6564888e14bbafa19bc887976786149e4f215b8e6d3038f", "https://esm.sh/boolbase@1.0.0/deno/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb", "https://esm.sh/boolbase@1.0.0/denonext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb", "https://esm.sh/boolbase@1.0.0/esnext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb", diff --git a/main.ts b/main.ts deleted file mode 100644 index ae47efe..0000000 --- a/main.ts +++ /dev/null @@ -1,333 +0,0 @@ -import path from 'node:path' -import { Buffer } from 'node:buffer' -import { parseArgs } from "jsr:@std/cli/parse-args" -import { exists } from "jsr:@std/fs/exists"; -import * as cheerio from 'https://esm.sh/cheerio?target=esnext' - -const REGEX_PLAYLISTURL = - /^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/?$/i -const REGEX_SONGURL = - /^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/[^\/]+\/?$/i -const REGEX_ALBUMTITLE = /^(.*?) MP3 - Download/i -const REGEX_UNSAFEFORFILE = /[^a-z0-9\-_=+,.()\[\]{} ]/gi - -// parse args -const flags = parseArgs(Deno.args, { - alias: { - "url": ["u"], - "output": ["o"], - "sync": ["s"], - "help": ["h"], - }, - string: ["url", "output"], - boolean: ["sync", "help"], - default: { - "url": "", - "output": ".", - "sync": false - } -}) - -if (!flags.url && flags._.length > 0) { - flags.url = flags._[0].toString() -} - -function printHelp() { - console.log(`deno - --allow-net --allow-write --allow-read main.ts - [--url] - [--output ] - [--sync] - [--help] - -parameters: - --url -u (default) - url to download - - --output -o - default: "." - output path - - --sync -s - download files one at a time - - --help -h - print help message -`) -} - -async function main() { - // load all song details - const playlist = await fetchPlaylist(flags.url) - - // if sync, download one at a time - if (flags.sync) { - console.log('downloading one at a time\n') - for (const song of playlist) { - await downloadSong(song, flags.output || '.') - } - } else { - console.log('downloading all at once\n') - await Promise.all( - playlist.map((song) => downloadSong(song, flags.output || '.')) - ) - } -} - -async function fetchPlaylist(url: string): Promise { - if (!REGEX_PLAYLISTURL.test(url)) { - throw `unaccepted url ${url}` - } - - console.log(`downloading: ${url}`) - - // load the playlist page's dom - const req = await fetch(url) - const text = await req.text() - const $ = cheerio.load(text) - - // get the album name from the page title - const title = - $.extract({ - title: 'title', - }).title ?? '' - const titleMatch = REGEX_ALBUMTITLE.exec(title) - if (titleMatch === null) { - throw `unable to grab album name from ${title}` - } - const albumName = titleMatch[1] - console.log(` title: ${albumName}`) - - // parse all rows in playlist - const columns = getPlaylistTableHeaders($) - const rows = getPlaylistRows($, columns, albumName) - console.log(` songs: ${rows.length}\n`) - - return rows -} - -interface PlaylistTableColumns { - name: number - track: number - cd: number -} - -interface PlaylistSongData { - url: string - album: string - name: string - track: number - cd: number -} - -function getPlaylistTableHeaders($: cheerio.CheerioAPI): PlaylistTableColumns { - // get table header row - const header = $('#songlist tr#songlist_header') - const headerCells = header.extract({ - cells: ['th'], - }) - - // get the index for specific columns - const indexes = headerCells.cells.reduce( - (p, c, i) => { - // check the string content of the current cell - switch (c.toLocaleLowerCase()) { - case 'cd': - p.cd = i - break - case '#': - p.track = i - break - case 'song name': - p.name = i - } - return p - }, - // default values - { - name: -1, - track: -1, - cd: -1, - } - ) - - if (indexes.name == -1) { - throw 'unable to find song title column' - } - - return indexes -} - -function getPlaylistRows( - $: cheerio.CheerioAPI, - columns: PlaylistTableColumns, - albumName: string -): PlaylistSongData[] { - const rows = $('#songlist tr:not(#songlist_header):not(#songlist_footer)') - const rowsData: PlaylistSongData[] = [] - - // loop through each song in table - rows.each((_, rowEl) => { - const row = cheerio.load(rowEl) - const rowData: PlaylistSongData = { - url: '', - album: albumName, - name: '', - track: -1, - cd: -1, - } - - // prase values from row - rowData.url = - row.extract({ - url: { - selector: 'td.playlistDownloadSong a', - value: 'href', - }, - }).url ?? '' - - const rowCells = row.extract({ - cells: ['td'], - }).cells - - rowData.name = rowCells[columns.name] - if (!rowData.name) { - throw `unable to grab song name from ${albumName} in row ${columns.name} - ${rowCells}` - } - - if (columns.track >= 0) { - rowData.track = parseInt(rowCells[columns.track] ?? '-1') - if (isNaN(rowData.track)) { - rowData.track = -1 - } - } - - if (columns.cd >= 0) { - rowData.cd = parseInt(rowCells[columns.cd] ?? '-1') - if (isNaN(rowData.cd)) { - rowData.cd = -1 - } - } - - rowsData.push(rowData) - }) - - return rowsData -} - -async function downloadSong(song: PlaylistSongData, location: string) { - // get full url - let url = song.url - if (!/^http/i.test(url)) { - url = 'https://downloads.khinsider.com' + url - } - if (!REGEX_SONGURL.test(url)) { - throw `unaccepted url ${url}` - } - - // load download page - const resp = await fetch(url) - const text = await resp.text() - const $ = cheerio.load(text) - - // extract the flac download link - const flacUrl = $.extract({ - url: { - selector: '#pageContent a[href*="flac"]', - value: 'href', - }, - }).url - - if (!flacUrl) { - throw `can't find download link for ${url}` - } - - // get the file and path to save the files - const { pathname, fullpathname } = pathFor(location, song) - - // ensure folder exists - if (!await exists(pathname)) { - await Deno.mkdir(pathname) - } - - // skip file if it exists - if (await exists(fullpathname)) { - console.log(`skipping file already exists ${fullpathname}`) - } - - console.log(`downloading ${fullpathname}`) - console.log(` from ${flacUrl}`) - - // download the file - const songresp = await fetch(flacUrl) - const songblob = await songresp.arrayBuffer() - return Deno.writeFile(fullpathname, toBuffer(songblob)) -} - -interface SongPath { - pathname: string - fullpathname: string -} - -function pathFor(location: string, song: PlaylistSongData): SongPath { - // clean strings for file paths - const albumname = song.album.replace(REGEX_UNSAFEFORFILE, '') - const songname = song.name.replace(REGEX_UNSAFEFORFILE, '') - - const cd = song.cd >= 0 ? song.cd + '.' : '' - const track = song.track >= 0 ? song.track + '.' : '' - const separator = song.cd >= 0 || song.track >= 0 ? ' ' : '' - const filename = `${cd}${track}${separator}${songname}.flac` - /* - for example - - song = { - songname: 'song', - track: 1, - cd: 1, - } - then filename = '1.1. song' - - song = { - songname: 'song', - track: 1, - cd: -1, - } - then filename = '1. song' - - song = { - songname: 'song', - track: -1, - cd: -1, - } - then filename = 'song' - */ - - const pathname = path.resolve(location, albumname) - const fullpathname = path.resolve(pathname, filename) - - return { - pathname, - fullpathname, - } -} - -// convert ArrayBuffer to Buffer -function toBuffer(arrayBuffer: ArrayBuffer): Buffer { - const buffer = Buffer.alloc(arrayBuffer.byteLength) - const view = new Uint8Array(arrayBuffer) - for (let i = 0; i < buffer.length; ++i) { - buffer[i] = view[i] - } - return buffer -} - -if (!flags.url) { - console.log('Missing URL\n\n') - printHelp() -} else if (flags.help) { - printHelp() -} else { - main().catch((e) => console.error(e)) -} - diff --git a/package.json b/package.json index 062e7f6..c419bed 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,5 @@ { + "main": "src/main.ts", "devDependencies": { "@types/node": "^24.5.2" } diff --git a/src/download.ts b/src/download.ts new file mode 100644 index 0000000..53a3119 --- /dev/null +++ b/src/download.ts @@ -0,0 +1,302 @@ +import EventEmitter from 'node:events' +import { DownloadFileDetails, randomChars } from './util.ts' + +export interface DownloadOptions { + name: string + url: URL + file?: DownloadFileDetails +} + +export interface DownloadOptionsID extends DownloadOptions { + id: string +} + +export interface DownloadItemStatus extends DownloadOptions { + downloaded: number + size?: number +} + +export interface DownloadStatus { + waiting: number + completed: number + active: DownloadItemStatus[] +} + +export interface DownloadQueueEvents { + update: (item: DownloadQueue) => void + done: (item: DownloadItem) => void + complete: () => void + error: (message: string) => void +} + +export declare interface DownloadQueue { + on( + event: U, + listener: DownloadQueueEvents[U] + ): this + on(event: `done_${string}`, listener: DownloadQueueEvents['done']): this + + once( + event: U, + listener: DownloadQueueEvents[U] + ): this + once(event: `done_${string}`, listener: DownloadQueueEvents['done']): this + + emit( + event: U, + ...args: Parameters + ): boolean + emit( + event: `done_${string}`, + ...args: Parameters + ): this +} + +export class DownloadQueue extends EventEmitter { + private queue_waiting: DownloadOptionsID[] = [] + private queue_active: DownloadItem[] = [] + private queue_history: DownloadItem[] = [] + private queue_size = 1 + private checking_queue = false + + constructor(queue_size: number) { + super() + if (queue_size > 1) { + this.queue_size = queue_size + } + } + + downloadBulk(...options: DownloadOptions[]): string[] { + const ids: string[] = [] + this.queue_waiting.push( + ...options.map((opt) => { + const id = this.generateID() + ids.push(id) + return { ...opt, id } + }) + ) + this.checkQueue() + + return ids + } + + download(options: DownloadOptions): string { + const id = this.generateID() + this.queue_waiting.push({ ...options, id }) + this.checkQueue() + + return id + } + + downloadPromise(options: DownloadOptions): Promise { + const id = this.download(options) + return new Promise((resolve) => { + this.once(`done_${id}`, (item) => { + resolve(item) + }) + }) + } + + complete(): Promise { + return new Promise((resolve) => { + this.on('complete', () => resolve()) + }) + } + + private IDExists(id: string): boolean { + const waitingIds = this.queue_waiting.map((item) => item.id) + const queuedIds = [...this.queue_active, ...this.queue_history].map( + (item) => item.options.id + ) + return [...waitingIds, ...queuedIds].some((itemId) => itemId === id) + } + + private generateID(): string { + let id = '' + do { + id = randomChars(16) + } while (this.IDExists(id)) + return id + } + + status(): DownloadStatus { + return { + waiting: this.queue_waiting.length, + completed: this.queue_history.length, + active: this.queue_active.map((item) => ({ + ...item.options, + downloaded: item.downloaded, + size: item.contentlength, + })), + } + } + + private checkQueue() { + if (this.checking_queue) { + return + } + + this.checking_queue = true + + // update active queue + const queue_active: DownloadItem[] = [] + for (const item of this.queue_active) { + if (item.completed) { + this.queue_history.push(item) + } else if (item.errored) { + this.queue_waiting.push(item.options) + } else { + queue_active.push(item) + } + } + this.queue_active = queue_active + + // add to active queue + while ( + this.queue_active.length < this.queue_size && + this.queue_waiting.length > 0 + ) { + const item = this.queue_waiting.shift() + if (!item) { + break + } + + this.queue_active.push(this.startDownloadItem(item)) + } + + this.checking_queue = false + this.emit('update', this) + + if (this.queue_active.length === 0 && this.queue_waiting.length === 0) { + this.emit('complete') + } + } + + private startDownloadItem(item: DownloadOptionsID): DownloadItem { + const download = new DownloadItem(item) + download.on('error', (err) => { + this.emit('error', err) + this.checkQueue() + }) + download.on('chunk', (_) => { + this.emit('update', this) + }) + download.on('done', (d) => { + this.emit('done', d) + this.emit(`done_${d.options.id}`, d) + this.checkQueue() + }) + return download + } +} + +export interface DownloadItemEvents { + chunk: (item: DownloadItem) => void + done: (item: DownloadItem) => void + error: (message: string) => void +} + +export declare interface DownloadItem { + on( + event: U, + listener: DownloadItemEvents[U] + ): this + once( + event: U, + listener: DownloadItemEvents[U] + ): this + emit( + event: U, + ...args: Parameters + ): boolean +} + +export class DownloadItem extends EventEmitter { + options: DownloadOptionsID + downloaded: number = 0 + contentlength?: number + content = new Uint8Array(0) + + completed = false + errored = false + + constructor(options: DownloadOptionsID) { + super() + this.options = options + this.startDownload() + .then(() => { + this.completed = true + this.emit('done', this) + }) + .catch((err) => { + this.errored = true + this.emit('error', err) + }) + } + + contentString(): string { + if (this.options.file || !this.completed) { + return '' + } + + const decoder = new TextDecoder('utf-8') + return decoder.decode(this.content) + } + + private updateContentLength(res: Response) { + const length = parseInt(res.headers.get('Content-Length') ?? '') + if (length && length > 0) { + this.contentlength = length + } + } + + private async startDownload() { + try { + const res = await fetch(this.options.url) + if (res.status !== 200) { + throw `HTTP status code: ${res.status} ${res.statusText}` + } + + if (!res.body) { + throw `HTTP response has no body` + } + + this.updateContentLength(res) + this.emit('chunk', this) + + if (this.options.file) { + await Deno.mkdir(this.options.file.fullpath, { + recursive: true, + }) + const file = await Deno.open(this.options.file.fullfilepath, { + create: true, + write: true, + }) + + for await (const chunk of res.body) { + await file.write(chunk) + this.downloaded += chunk.length + + this.emit('chunk', this) + } + } else { + const chunks = [] + for await (const chunk of res.body) { + chunks.push(chunk) + this.downloaded += chunk.length + + this.emit('chunk', this) + } + + this.content = new Uint8Array(this.downloaded) + let offset = 0 + for (const chunk of chunks) { + this.content.set(chunk, offset) + offset += chunk.length + } + } + } catch (e) { + throw e + } + } +} diff --git a/src/main.ts b/src/main.ts new file mode 100644 index 0000000..4be516c --- /dev/null +++ b/src/main.ts @@ -0,0 +1,108 @@ +import { parseArgs } from 'jsr:@std/cli/parse-args' +import { downloadSong, fetchPlaylist } from './web.ts' +import { DownloadOptions, DownloadQueue } from './download.ts' +import { UI } from './ui.ts' +import { getFileDetails } from './util.ts' + +// parse args +const flags = parseArgs(Deno.args, { + alias: { + url: ['u'], + output: ['o'], + queuesize: ['qs'], + help: ['h'], + }, + string: ['url', 'output', 'queuesize'], + boolean: ['help'], + default: { + url: '', + output: '.', + queuesize: '5', + }, +}) + +if (!flags.url && flags._.length > 0) { + flags.url = flags._[0].toString() +} + +const downloadQueueSize = parseInt(flags.queuesize) +if (!downloadQueueSize || downloadQueueSize < 1) { + console.log(`Unable to parse parse Queue Size: "${flags.queuesize}"`) + Deno.exit() +} + +function printHelp() { + console.log(`deno + --allow-net --allow-write --allow-read main.ts + [--url] + [--output ] + [--queuesize ] + [--help] + +parameters: + [--url] + alis: -u + required: true + url to download + + --output + alias: -o + default: "." + output path + + --queuesize + alias: -qs + default: 5 + number of files to download simultaneously + + --help + alias: -h + print help message +`) +} + +async function main() { + const downloadQueue = new DownloadQueue(5) + const ui = new UI(downloadQueue) + + // load all song details + const playlistUrl = new URL(flags.url) + const { playlist, images } = await fetchPlaylist( + ui, + downloadQueue, + playlistUrl + ) + + const albumName = playlist[0].album + + const imageDownloadOptions: DownloadOptions[] = images.map( + (imageUrl, i) => { + const imagename = decodeURIComponent( + imageUrl.pathname.split('/').pop() ?? `image_${i}` + ) + const file = getFileDetails(flags.output, albumName, imagename) + return { + url: imageUrl, + name: `Image - ${file.file}`, + file, + } + } + ) + downloadQueue.downloadBulk(...imageDownloadOptions) + + for (const song of playlist) { + downloadSong(ui, downloadQueue, song, flags.output) + } + + await downloadQueue.complete() + ui.printLogs() +} + +if (!flags.url) { + console.log('Missing URL\n\n') + printHelp() +} else if (flags.help) { + printHelp() +} else { + main().catch((e) => console.error(e)) +} diff --git a/src/ui.ts b/src/ui.ts new file mode 100644 index 0000000..2de9925 --- /dev/null +++ b/src/ui.ts @@ -0,0 +1,120 @@ +import { DownloadQueue } from './download.ts' +import { getHeight, getWidth } from 'https://deno.land/x/terminal_size/mod.ts' +import { humanizeBytes, round } from './util.ts' +import { stdout } from 'node:process' + +interface LogItem { + date: Date + message: string +} + +export class UI { + private downloads: DownloadQueue + private log_history: LogItem[] = [] + + constructor(downloads: DownloadQueue) { + this.downloads = downloads + this.printDownloads() + downloads.on('update', (_) => this.printDownloads()) + } + + // deno-lint-ignore no-explicit-any + log(...message: any[]) { + const msg = message.map((m) => m.toString()).join(' ') + const date = new Date() + this.log_history.push({ date, message: msg }) + this.printDownloads() + } + + // deno-lint-ignore no-explicit-any + logExit(...message: any[]) { + this.log(...message) + this.printLogs() + Deno.exit() + } + + printLogs() { + for (const logItem of this.log_history) { + const dateStr = logItem.date.toISOString() + console.log(`${dateStr} ${logItem.message}`) + } + } + + private printDownloads() { + const terminalWidth = getWidth() + const terminaHeight = getHeight() + // console.log(terminalWidth, terminaHeight) + // process.exit() + const status = this.downloads.status() + let messageLines: string[] = [] + + // active queue + for (const item of status.active) { + const downloaded = humanizeBytes(item.downloaded) + const total = item.size ? humanizeBytes(item.size) : '???' + const percent = item.size + ? round((item.downloaded / item.size) * 100, 0) + : '??' + + { + let nameLine = `${item.name}` + const nameLineEnd = ` ${downloaded} / ${total} (${percent}%)` + + const nameLineSpaces = + terminalWidth - (nameLine.length + nameLineEnd.length) + if (nameLineSpaces >= 0) { + const spaces = new Array(nameLineSpaces).fill(' ').join('') + nameLine = nameLine + spaces + nameLineEnd + } else { + nameLine = + nameLine.slice(0, nameLine.length + nameLineSpaces) + + nameLineEnd + } + + messageLines.push(nameLine) + } + + messageLines.push(` from: ${item.url.pathname}`) + messageLines.push( + ` to: ${item.file ? item.file.relpath : 'Memory Buffer'}` + ) + } + + // other details + messageLines.push(`___`) + messageLines.push( + `Active/Waiting/Completed = ${status.active.length}/${status.waiting}/${status.completed}` + ) + + // trim away excess lines + const messageStartIndex = messageLines.length - terminaHeight + if (messageStartIndex >= 0) { + messageLines = messageLines.slice(messageStartIndex) + } else { + messageLines = [ + ...new Array(-messageStartIndex).fill(''), + ...messageLines, + ] + } + + // trim excess line widths + messageLines = messageLines.map((line) => { + if (line.length >= terminalWidth) { + return line.slice(0, terminalWidth) + } else { + const spaces = new Array(terminalWidth - line.length) + .fill(' ') + .join('') + return line + spaces + } + // return line + }) + + // clear screen, go to top left + // stdout.write("\x1b[2J") + stdout.write(`\x1b[0;0H`) + + // print lines + stdout.write(messageLines.join('\n')) + } +} diff --git a/src/util.ts b/src/util.ts new file mode 100644 index 0000000..fe8aa42 --- /dev/null +++ b/src/util.ts @@ -0,0 +1,195 @@ +import { resolve, join } from 'node:path' + +const REGEX_UNSAFEFORFILE = /[\\\/:*?'"<>|*!?]|^null$|^\./gi + +/** example + * { + * path: 'foldername', + * fullpath: '/path/to/foldername', + * location: '/path/to', + * + * file: 'filename', + * ext: 'ext', + * fullfile: 'filename.ext + * relpath: 'foldername/filename.ext', + * fullfilepath: '/path/to/foldername/filename.ext', + * } + */ +export interface DownloadFolderDetails { + path: string + fullpath: string + location: string +} +export interface DownloadFileDetails extends DownloadFolderDetails { + file: string + ext: string + fullfile: string + relpath: string + fullfilepath: string +} + +export interface PlaylistSongData { + url: URL + album: string + name: string + track: number + cd: number +} + +export function songFoldername( + song: Partial | string +): string { + if (typeof song != 'string') { + song = song.album ?? '' + } + + const albumname = song.replace(REGEX_UNSAFEFORFILE, '') + return albumname +} + +export function songFilename( + song: Partial +): [filename: string, extension: string] { + if (!song.name) { + return ['', ''] + } + + const songname = song.name.replace(REGEX_UNSAFEFORFILE, '') + const tracknum = song.track ?? -1 + const cdnum = song.cd ?? -1 + + const cd = cdnum >= 0 ? cdnum + '.' : '' + const track = tracknum >= 0 ? tracknum + '.' : '' + const separator = cdnum >= 0 || tracknum >= 0 ? ' ' : '' + + return [`${cd}${track}${separator}${songname}`, 'flac'] + + /* + for example + + song = { + name: 'song', + track: 1, + cd: 1, + } + then filename = '1.1. song' + + song = { + name: 'song', + track: 1, + cd: -1, + } + then filename = '1. song' + + song = { + name: 'song', + track: -1, + cd: -1, + } + then filename = 'song' + */ +} + +export function splitFilename( + filename: string +): [filename: string, extension: string] { + const spl = filename.split('.') + let ext = 'bin' + if (spl.length > 1) { + ext = spl.pop() ?? ext + } + return [spl.join('.'), ext] +} + +export function getSongFileDetails( + location: string, + song: Partial +): DownloadFileDetails { + const path = songFoldername(song) + const [file, ext] = songFilename(song) + const fullfile = `${file}.${ext}` + + const relpath = join(path, fullfile) + + const fullpath = resolve(location, path) + const fullfilepath = resolve(fullpath, fullfile) + + return { + path, + file, + ext, + fullfile, + relpath, + fullpath, + fullfilepath, + location, + } +} + +export function getFileDetails( + location: string, + albumName: string, + filename: string +): DownloadFileDetails { + const path = songFoldername(albumName) + + const [file, ext] = splitFilename(filename) + const fullfile = `${file}.${ext}` + + const relpath = join(path, fullfile) + + const fullpath = resolve(location, path) + const fullfilepath = resolve(fullpath, fullfile) + + return { + path, + file, + ext, + fullfile, + relpath, + fullpath, + fullfilepath, + location, + } +} + +const BYTE_MAGNITUDE = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] +export function humanizeBytes(size: number, decimals: number = 2) { + let magnitude = 0 + while (size > 1000) { + magnitude++ + size /= 1000 + } + + size = round(size, decimals) + + let magnitudeString = '' + if (magnitude < BYTE_MAGNITUDE.length) { + magnitudeString = BYTE_MAGNITUDE[magnitude] + } else { + const excessMagnitude = magnitude - BYTE_MAGNITUDE.length + magnitudeString = `${ + BYTE_MAGNITUDE[BYTE_MAGNITUDE.length - 1] + }+${excessMagnitude}` + } + + return `${size}${magnitudeString}` +} + +export function round(number: number, decimals: number = 2) { + decimals = Math.max(0, Math.round(decimals)) + const decimalsMagnitude = Math.pow(10, decimals) + return Math.round(number * decimalsMagnitude) / decimalsMagnitude +} + +const CHARS = + 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.split('') +export function randomChar() { + return CHARS[Math.round(Math.random() * (CHARS.length - 1))] +} +export function randomChars(length: number) { + return new Array(length) + .fill('') + .map((_) => randomChar()) + .join('') +} diff --git a/src/web.ts b/src/web.ts new file mode 100644 index 0000000..63c3585 --- /dev/null +++ b/src/web.ts @@ -0,0 +1,241 @@ +import { exists } from 'jsr:@std/fs/exists' +import * as cheerio from 'https://esm.sh/cheerio?target=esnext' +import { DownloadQueue } from './download.ts' +import { UI } from './ui.ts' +import { + DownloadFileDetails, + getSongFileDetails, + PlaylistSongData, +} from './util.ts' + +const REGEX_PLAYLISTURL = + /^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/?$/i +const REGEX_SONGURL = + /^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/[^\/]+\/?$/i +const REGEX_ALBUMTITLE = /^(.*?) MP3 - Download/i + +export interface PlaylistWhole { + playlist: PlaylistSongData[] + images: URL[] +} + +export async function fetchPlaylist( + ui: UI, + downloadQueue: DownloadQueue, + url: URL +): Promise { + if (!REGEX_PLAYLISTURL.test(url.toString())) { + throw `unaccepted url ${url}` + } + + ui.log(`downloading: ${url}`) + + // load the playlist page's dom + const req = await downloadQueue.downloadPromise({ + name: 'Playlist HTML', + url, + }) + const text = req.contentString() + const $ = cheerio.load(text) + + // get the album name from the page title + const title = + $.extract({ + title: 'title', + }).title ?? '' + const titleMatch = REGEX_ALBUMTITLE.exec(title) + if (titleMatch === null) { + throw `unable to grab album name from ${title}` + } + const albumName = titleMatch[1] + ui.log(` title: ${albumName}`) + + // get the album images from the page + const images = + $.extract({ + images: [ + { + selector: '.albumImage a', + value: 'href', + }, + ], + }).images ?? [] + const imageUrls = images.map((img) => new URL(img)) + ui.log(` images: ${images.length}`) + + // parse all rows in playlist + const columns = getPlaylistTableHeaders($) + const rows = getPlaylistRows($, columns, albumName) + ui.log(` songs: ${rows.length}\n`) + + return { playlist: rows, images: imageUrls } +} + +interface PlaylistTableColumns { + name: number + track: number + cd: number +} + +function getPlaylistTableHeaders($: cheerio.CheerioAPI): PlaylistTableColumns { + // get table header row + const header = $('#songlist tr#songlist_header') + const headerCells = header.extract({ + cells: ['th'], + }) + + // get the index for specific columns + const indexes = headerCells.cells.reduce( + (p, c, i) => { + // check the string content of the current cell + switch (c.toLocaleLowerCase()) { + case 'cd': + p.cd = i + break + case '#': + p.track = i + break + case 'song name': + p.name = i + } + return p + }, + // default values + { + name: -1, + track: -1, + cd: -1, + } + ) + + if (indexes.name == -1) { + throw 'unable to find song title column' + } + + return indexes +} + +function getPlaylistRows( + $: cheerio.CheerioAPI, + columns: PlaylistTableColumns, + albumName: string +): PlaylistSongData[] { + const rows = $('#songlist tr:not(#songlist_header):not(#songlist_footer)') + const rowsData: PlaylistSongData[] = [] + + // loop through each song in table + rows.each((_, rowEl) => { + const row = cheerio.load(rowEl) + + // prase values from row + const urlStr = + row.extract({ + url: { + selector: 'td.playlistDownloadSong a', + value: 'href', + }, + }).url ?? '' + + const rowData: PlaylistSongData = { + url: newKHInsiderURL(urlStr), + album: albumName, + name: '', + track: -1, + cd: -1, + } + + const rowCells = row.extract({ + cells: ['td'], + }).cells + + rowData.name = rowCells[columns.name] + if (!rowData.name) { + throw `unable to grab song name from ${albumName} in row ${columns.name} - ${rowCells}` + } + + if (columns.track >= 0) { + rowData.track = parseInt(rowCells[columns.track] ?? '-1') + if (isNaN(rowData.track)) { + rowData.track = -1 + } + } + + if (columns.cd >= 0) { + rowData.cd = parseInt(rowCells[columns.cd] ?? '-1') + if (isNaN(rowData.cd)) { + rowData.cd = -1 + } + } + + rowsData.push(rowData) + }) + + return rowsData +} + +function newKHInsiderURL(url: string) { + if (!/^http/i.test(url.toString())) { + return new URL('https://downloads.khinsider.com' + url) + } + return new URL(url) +} + +export async function downloadSong( + ui: UI, + downloadQueue: DownloadQueue, + song: PlaylistSongData, + location: string +) { + // get full url + if (!REGEX_SONGURL.test(song.url.toString())) { + throw `unaccepted url ${song.url}` + } + + const file = getSongFileDetails(location, song) + + // skip file if it exists + if (await exists(file.fullfilepath)) { + ui.log(`skipping file already exists ${file.fullfilepath}`) + return + } + + // get file url + const flacURL = await getFileURL(downloadQueue, song, file) + + ui.log(`downloading ${file.relpath}`) + ui.log(` from ${flacURL}`) + + // download the file + downloadQueue.download({ + url: flacURL, + name: `File - ${file.file}`, + file, + }) +} + +async function getFileURL( + downloadQueue: DownloadQueue, + song: PlaylistSongData, + file: DownloadFileDetails +) { + // load download page + const resp = await downloadQueue.downloadPromise({ + url: song.url, + name: `DL Page - ${file.file}`, + }) + const text = resp.contentString() + const $ = cheerio.load(text) + + // extract the flac download link + const flacUrlStr = $.extract({ + url: { + selector: '#pageContent a[href*="flac"]', + value: 'href', + }, + }).url + if (!flacUrlStr) { + throw `can't find download link for ${song.url}` + } + + return new URL(flacUrlStr) +}