zomo/khinsider-downloads
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

created a download queue, added full screen status viewing

Browse Source
This commit is contained in:
zomo
2025-10-24 01:59:35 -05:00
parent fa199ea1bb
commit 1545cf9398
9 changed files with 1013 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.prettierrc
View File

@@ -1,7 +1,6 @@
{ {
"tabWidth": 2, "tabWidth": 4,
"useTabs": false, "useTabs": false,
"semi": false, "semi": false,
"singleQuote": true, "singleQuote": true
"objectWrap": "collapse"
} }

44
deno.lock generated
View File

@@ -31,6 +31,7 @@
} }
}, },
"redirects": { "redirects": {
"https://deno.land/x/terminal_size/mod.ts": "https://deno.land/x/terminal_size@0.1.0/mod.ts",
"https://esm.sh/@types/boolbase@~1.0.3/index.d.ts": "https://esm.sh/@types/boolbase@1.0.3/index.d.ts", "https://esm.sh/@types/boolbase@~1.0.3/index.d.ts": "https://esm.sh/@types/boolbase@1.0.3/index.d.ts",
"https://esm.sh/@types/safer-buffer@~2.1.3/index.d.ts": "https://esm.sh/@types/safer-buffer@2.1.3/index.d.ts", "https://esm.sh/@types/safer-buffer@~2.1.3/index.d.ts": "https://esm.sh/@types/safer-buffer@2.1.3/index.d.ts",
"https://esm.sh/boolbase@^1.0.0?target=deno": "https://esm.sh/boolbase@1.0.0?target=deno", "https://esm.sh/boolbase@^1.0.0?target=deno": "https://esm.sh/boolbase@1.0.0?target=deno",
@@ -89,6 +90,18 @@
"https://esm.sh/whatwg-mimetype@^4.0.0?target=denonext": "https://esm.sh/whatwg-mimetype@4.0.0?target=denonext" "https://esm.sh/whatwg-mimetype@^4.0.0?target=denonext": "https://esm.sh/whatwg-mimetype@4.0.0?target=denonext"
}, },
"remote": { "remote": {
"https://deno.land/std@0.122.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58",
"https://deno.land/std@0.122.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac",
"https://deno.land/std@0.122.0/fmt/colors.ts": "8368ddf2d48dfe413ffd04cdbb7ae6a1009cf0dccc9c7ff1d76259d9c61a0621",
"https://deno.land/std@0.122.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853",
"https://deno.land/std@0.122.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4",
"https://deno.land/std@0.122.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b",
"https://deno.land/std@0.122.0/path/common.ts": "f41a38a0719a1e85aa11c6ba3bea5e37c15dd009d705bd8873f94c833568cbc4",
"https://deno.land/std@0.122.0/path/glob.ts": "7bf2349e818e332a830f3d8874c3f45dd7580b6c742ed50dbf6282d84ab18405",
"https://deno.land/std@0.122.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12",
"https://deno.land/std@0.122.0/path/posix.ts": "34349174b9cd121625a2810837a82dd8b986bbaaad5ade690d1de75bbb4555b2",
"https://deno.land/std@0.122.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c",
"https://deno.land/std@0.122.0/path/win32.ts": "11549e8c6df8307a8efcfa47ad7b2a75da743eac7d4c89c9723a944661c8bd2e",
"https://deno.land/std@0.177.1/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", "https://deno.land/std@0.177.1/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462",
"https://deno.land/std@0.177.1/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", "https://deno.land/std@0.177.1/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3",
"https://deno.land/std@0.177.1/async/abortable.ts": "73acfb3ed7261ce0d930dbe89e43db8d34e017b063cf0eaa7d215477bf53442e", "https://deno.land/std@0.177.1/async/abortable.ts": "73acfb3ed7261ce0d930dbe89e43db8d34e017b063cf0eaa7d215477bf53442e",
@@ -377,6 +390,37 @@
"https://deno.land/std@0.177.1/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", "https://deno.land/std@0.177.1/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea",
"https://deno.land/std@0.177.1/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", "https://deno.land/std@0.177.1/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7",
"https://deno.land/std@0.177.1/testing/asserts.ts": "984ab0bfb3faeed92ffaa3a6b06536c66811185328c5dd146257c702c41b01ab", "https://deno.land/std@0.177.1/testing/asserts.ts": "984ab0bfb3faeed92ffaa3a6b06536c66811185328c5dd146257c702c41b01ab",
"https://deno.land/std@0.97.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58",
"https://deno.land/std@0.97.0/_util/os.ts": "e282950a0eaa96760c0cf11e7463e66babd15ec9157d4c9ed49cc0925686f6a7",
"https://deno.land/std@0.97.0/encoding/base64.ts": "eecae390f1f1d1cae6f6c6d732ede5276bf4b9cd29b1d281678c054dc5cc009e",
"https://deno.land/std@0.97.0/encoding/hex.ts": "f952e0727bddb3b2fd2e6889d104eacbd62e92091f540ebd6459317a61932d9b",
"https://deno.land/std@0.97.0/fs/_util.ts": "f2ce811350236ea8c28450ed822a5f42a0892316515b1cd61321dec13569c56b",
"https://deno.land/std@0.97.0/fs/ensure_dir.ts": "b7c103dc41a3d1dbbb522bf183c519c37065fdc234831a4a0f7d671b1ed5fea7",
"https://deno.land/std@0.97.0/fs/exists.ts": "b0d2e31654819cc2a8d37df45d6b14686c0cc1d802e9ff09e902a63e98b85a00",
"https://deno.land/std@0.97.0/hash/_wasm/hash.ts": "cb6ad1ab429f8ac9d6eae48f3286e08236d662e1a2e5cfd681ba1c0f17375895",
"https://deno.land/std@0.97.0/hash/_wasm/wasm.js": "94b1b997ae6fb4e6d2156bcea8f79cfcd1e512a91252b08800a92071e5e84e1a",
"https://deno.land/std@0.97.0/hash/hasher.ts": "57a9ec05dd48a9eceed319ac53463d9873490feea3832d58679df6eec51c176b",
"https://deno.land/std@0.97.0/hash/mod.ts": "5d032bd34186cda2f8d17fc122d621430953a6030d4b3f11172004715e3e2441",
"https://deno.land/std@0.97.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853",
"https://deno.land/std@0.97.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4",
"https://deno.land/std@0.97.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b",
"https://deno.land/std@0.97.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a",
"https://deno.land/std@0.97.0/path/glob.ts": "314ad9ff263b895795208cdd4d5e35a44618ca3c6dd155e226fb15d065008652",
"https://deno.land/std@0.97.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12",
"https://deno.land/std@0.97.0/path/posix.ts": "f56c3c99feb47f30a40ce9d252ef6f00296fa7c0fcb6dd81211bdb3b8b99ca3b",
"https://deno.land/std@0.97.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c",
"https://deno.land/std@0.97.0/path/win32.ts": "77f7b3604e0de40f3a7c698e8a79e7f601dc187035a1c21cb1e596666ce112f8",
"https://deno.land/x/cache@0.2.13/cache.ts": "4005aad54fb9aac9ff02526ffa798032e57f2d7966905fdeb7949263b1c95f2f",
"https://deno.land/x/cache@0.2.13/deps.ts": "6f14e76a1a09f329e3f3830c6e72bd10b53a89a75769d5ea886e5d8603e503e6",
"https://deno.land/x/cache@0.2.13/directories.ts": "ef48531cab3f827252e248596d15cede0de179a2fb15392ae24cf8034519994f",
"https://deno.land/x/cache@0.2.13/file.ts": "5abe7d80c6ac594c98e66eb4262962139f48cd9c49dbe2a77e9608760508a09a",
"https://deno.land/x/cache@0.2.13/file_fetcher.ts": "5c793cc83a5b9377679ec313b2a2321e51bf7ed15380fa82d387f1cdef3b924f",
"https://deno.land/x/cache@0.2.13/helpers.ts": "d1545d6432277b7a0b5ea254d1c51d572b6452a8eadd9faa7ad9c5586a1725c4",
"https://deno.land/x/cache@0.2.13/mod.ts": "3188250d3a013ef6c9eb060e5284cf729083af7944a29e60bb3d8597dd20ebcd",
"https://deno.land/x/plug@0.5.1/deps.ts": "0f53866f60dc4f89bbc3be9d096f7501f2068a51923db220f43f0ad284b6b892",
"https://deno.land/x/plug@0.5.1/plug.ts": "620f9454df049228ba715b215582200a344f33f37605d5b4dfce5bf89c7e9815",
"https://deno.land/x/terminal_size@0.1.0/bindings/bindings.ts": "712ba91bfc7c93d5bbad2e6c996ccc062d9a12ac3c0fa258ed068f666ce43d3b",
"https://deno.land/x/terminal_size@0.1.0/mod.ts": "6ab2530394db2675f6564888e14bbafa19bc887976786149e4f215b8e6d3038f",
"https://esm.sh/boolbase@1.0.0/deno/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb", "https://esm.sh/boolbase@1.0.0/deno/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb",
"https://esm.sh/boolbase@1.0.0/denonext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb", "https://esm.sh/boolbase@1.0.0/denonext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb",
"https://esm.sh/boolbase@1.0.0/esnext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb", "https://esm.sh/boolbase@1.0.0/esnext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb",

333
main.ts
View File

@@ -1,333 +0,0 @@
import path from 'node:path'
import { Buffer } from 'node:buffer'
import { parseArgs } from "jsr:@std/cli/parse-args"
import { exists } from "jsr:@std/fs/exists";
import * as cheerio from 'https://esm.sh/cheerio?target=esnext'
const REGEX_PLAYLISTURL =
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/?$/i
const REGEX_SONGURL =
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/[^\/]+\/?$/i
const REGEX_ALBUMTITLE = /^(.*?) MP3 - Download/i
const REGEX_UNSAFEFORFILE = /[^a-z0-9\-_=+,.()\[\]{} ]/gi
// parse args
const flags = parseArgs(Deno.args, {
alias: {
"url": ["u"],
"output": ["o"],
"sync": ["s"],
"help": ["h"],
},
string: ["url", "output"],
boolean: ["sync", "help"],
default: {
"url": "",
"output": ".",
"sync": false
}
})
if (!flags.url && flags._.length > 0) {
flags.url = flags._[0].toString()
}
function printHelp() {
console.log(`deno
--allow-net --allow-write --allow-read main.ts
[--url] <url>
[--output <downloadPath>]
[--sync]
[--help]
parameters:
--url -u (default)
url to download
--output -o
default: "."
output path
--sync -s
download files one at a time
--help -h
print help message
`)
}
async function main() {
// load all song details
const playlist = await fetchPlaylist(flags.url)
// if sync, download one at a time
if (flags.sync) {
console.log('downloading one at a time\n')
for (const song of playlist) {
await downloadSong(song, flags.output || '.')
}
} else {
console.log('downloading all at once\n')
await Promise.all(
playlist.map((song) => downloadSong(song, flags.output || '.'))
)
}
}
async function fetchPlaylist(url: string): Promise<PlaylistSongData[]> {
if (!REGEX_PLAYLISTURL.test(url)) {
throw `unaccepted url ${url}`
}
console.log(`downloading: ${url}`)
// load the playlist page's dom
const req = await fetch(url)
const text = await req.text()
const $ = cheerio.load(text)
// get the album name from the page title
const title =
$.extract({
title: 'title',
}).title ?? ''
const titleMatch = REGEX_ALBUMTITLE.exec(title)
if (titleMatch === null) {
throw `unable to grab album name from ${title}`
}
const albumName = titleMatch[1]
console.log(` title: ${albumName}`)
// parse all rows in playlist
const columns = getPlaylistTableHeaders($)
const rows = getPlaylistRows($, columns, albumName)
console.log(` songs: ${rows.length}\n`)
return rows
}
interface PlaylistTableColumns {
name: number
track: number
cd: number
}
interface PlaylistSongData {
url: string
album: string
name: string
track: number
cd: number
}
function getPlaylistTableHeaders($: cheerio.CheerioAPI): PlaylistTableColumns {
// get table header row
const header = $('#songlist tr#songlist_header')
const headerCells = header.extract({
cells: ['th'],
})
// get the index for specific columns
const indexes = headerCells.cells.reduce(
(p, c, i) => {
// check the string content of the current cell
switch (c.toLocaleLowerCase()) {
case 'cd':
p.cd = i
break
case '#':
p.track = i
break
case 'song name':
p.name = i
}
return p
},
// default values
{
name: -1,
track: -1,
cd: -1,
}
)
if (indexes.name == -1) {
throw 'unable to find song title column'
}
return indexes
}
function getPlaylistRows(
$: cheerio.CheerioAPI,
columns: PlaylistTableColumns,
albumName: string
): PlaylistSongData[] {
const rows = $('#songlist tr:not(#songlist_header):not(#songlist_footer)')
const rowsData: PlaylistSongData[] = []
// loop through each song in table
rows.each((_, rowEl) => {
const row = cheerio.load(rowEl)
const rowData: PlaylistSongData = {
url: '',
album: albumName,
name: '',
track: -1,
cd: -1,
}
// prase values from row
rowData.url =
row.extract({
url: {
selector: 'td.playlistDownloadSong a',
value: 'href',
},
}).url ?? ''
const rowCells = row.extract({
cells: ['td'],
}).cells
rowData.name = rowCells[columns.name]
if (!rowData.name) {
throw `unable to grab song name from ${albumName} in row ${columns.name} - ${rowCells}`
}
if (columns.track >= 0) {
rowData.track = parseInt(rowCells[columns.track] ?? '-1')
if (isNaN(rowData.track)) {
rowData.track = -1
}
}
if (columns.cd >= 0) {
rowData.cd = parseInt(rowCells[columns.cd] ?? '-1')
if (isNaN(rowData.cd)) {
rowData.cd = -1
}
}
rowsData.push(rowData)
})
return rowsData
}
async function downloadSong(song: PlaylistSongData, location: string) {
// get full url
let url = song.url
if (!/^http/i.test(url)) {
url = 'https://downloads.khinsider.com' + url
}
if (!REGEX_SONGURL.test(url)) {
throw `unaccepted url ${url}`
}
// load download page
const resp = await fetch(url)
const text = await resp.text()
const $ = cheerio.load(text)
// extract the flac download link
const flacUrl = $.extract({
url: {
selector: '#pageContent a[href*="flac"]',
value: 'href',
},
}).url
if (!flacUrl) {
throw `can't find download link for ${url}`
}
// get the file and path to save the files
const { pathname, fullpathname } = pathFor(location, song)
// ensure folder exists
if (!await exists(pathname)) {
await Deno.mkdir(pathname)
}
// skip file if it exists
if (await exists(fullpathname)) {
console.log(`skipping file already exists ${fullpathname}`)
}
console.log(`downloading ${fullpathname}`)
console.log(` from ${flacUrl}`)
// download the file
const songresp = await fetch(flacUrl)
const songblob = await songresp.arrayBuffer()
return Deno.writeFile(fullpathname, toBuffer(songblob))
}
interface SongPath {
pathname: string
fullpathname: string
}
function pathFor(location: string, song: PlaylistSongData): SongPath {
// clean strings for file paths
const albumname = song.album.replace(REGEX_UNSAFEFORFILE, '')
const songname = song.name.replace(REGEX_UNSAFEFORFILE, '')
const cd = song.cd >= 0 ? song.cd + '.' : ''
const track = song.track >= 0 ? song.track + '.' : ''
const separator = song.cd >= 0 || song.track >= 0 ? ' ' : ''
const filename = `${cd}${track}${separator}${songname}.flac`
/*
for example
song = {
songname: 'song',
track: 1,
cd: 1,
}
then filename = '1.1. song'
song = {
songname: 'song',
track: 1,
cd: -1,
}
then filename = '1. song'
song = {
songname: 'song',
track: -1,
cd: -1,
}
then filename = 'song'
*/
const pathname = path.resolve(location, albumname)
const fullpathname = path.resolve(pathname, filename)
return {
pathname,
fullpathname,
}
}
// convert ArrayBuffer to Buffer<ArrayBuffer>
function toBuffer(arrayBuffer: ArrayBuffer): Buffer<ArrayBuffer> {
const buffer = Buffer.alloc(arrayBuffer.byteLength)
const view = new Uint8Array(arrayBuffer)
for (let i = 0; i < buffer.length; ++i) {
buffer[i] = view[i]
}
return buffer
}
if (!flags.url) {
console.log('Missing URL\n\n')
printHelp()
} else if (flags.help) {
printHelp()
} else {
main().catch((e) => console.error(e))
}

1
package.json
View File

@@ -1,4 +1,5 @@
{ {
"main": "src/main.ts",
"devDependencies": { "devDependencies": {
"@types/node": "^24.5.2" "@types/node": "^24.5.2"
} }

302
src/download.ts Normal file
View File

@@ -0,0 +1,302 @@
import EventEmitter from 'node:events'
import { DownloadFileDetails, randomChars } from './util.ts'
export interface DownloadOptions {
name: string
url: URL
file?: DownloadFileDetails
}
export interface DownloadOptionsID extends DownloadOptions {
id: string
}
export interface DownloadItemStatus extends DownloadOptions {
downloaded: number
size?: number
}
export interface DownloadStatus {
waiting: number
completed: number
active: DownloadItemStatus[]
}
export interface DownloadQueueEvents {
update: (item: DownloadQueue) => void
done: (item: DownloadItem) => void
complete: () => void
error: (message: string) => void
}
export declare interface DownloadQueue {
on<U extends keyof DownloadQueueEvents>(
event: U,
listener: DownloadQueueEvents[U]
): this
on(event: `done_${string}`, listener: DownloadQueueEvents['done']): this
once<U extends keyof DownloadQueueEvents>(
event: U,
listener: DownloadQueueEvents[U]
): this
once(event: `done_${string}`, listener: DownloadQueueEvents['done']): this
emit<U extends keyof DownloadQueueEvents>(
event: U,
...args: Parameters<DownloadQueueEvents[U]>
): boolean
emit(
event: `done_${string}`,
...args: Parameters<DownloadQueueEvents['done']>
): this
}
export class DownloadQueue extends EventEmitter {
private queue_waiting: DownloadOptionsID[] = []
private queue_active: DownloadItem[] = []
private queue_history: DownloadItem[] = []
private queue_size = 1
private checking_queue = false
constructor(queue_size: number) {
super()
if (queue_size > 1) {
this.queue_size = queue_size
}
}
downloadBulk(...options: DownloadOptions[]): string[] {
const ids: string[] = []
this.queue_waiting.push(
...options.map((opt) => {
const id = this.generateID()
ids.push(id)
return { ...opt, id }
})
)
this.checkQueue()
return ids
}
download(options: DownloadOptions): string {
const id = this.generateID()
this.queue_waiting.push({ ...options, id })
this.checkQueue()
return id
}
downloadPromise(options: DownloadOptions): Promise<DownloadItem> {
const id = this.download(options)
return new Promise((resolve) => {
this.once(`done_${id}`, (item) => {
resolve(item)
})
})
}
complete(): Promise<void> {
return new Promise((resolve) => {
this.on('complete', () => resolve())
})
}
private IDExists(id: string): boolean {
const waitingIds = this.queue_waiting.map((item) => item.id)
const queuedIds = [...this.queue_active, ...this.queue_history].map(
(item) => item.options.id
)
return [...waitingIds, ...queuedIds].some((itemId) => itemId === id)
}
private generateID(): string {
let id = ''
do {
id = randomChars(16)
} while (this.IDExists(id))
return id
}
status(): DownloadStatus {
return {
waiting: this.queue_waiting.length,
completed: this.queue_history.length,
active: this.queue_active.map((item) => ({
...item.options,
downloaded: item.downloaded,
size: item.contentlength,
})),
}
}
private checkQueue() {
if (this.checking_queue) {
return
}
this.checking_queue = true
// update active queue
const queue_active: DownloadItem[] = []
for (const item of this.queue_active) {
if (item.completed) {
this.queue_history.push(item)
} else if (item.errored) {
this.queue_waiting.push(item.options)
} else {
queue_active.push(item)
}
}
this.queue_active = queue_active
// add to active queue
while (
this.queue_active.length < this.queue_size &&
this.queue_waiting.length > 0
) {
const item = this.queue_waiting.shift()
if (!item) {
break
}
this.queue_active.push(this.startDownloadItem(item))
}
this.checking_queue = false
this.emit('update', this)
if (this.queue_active.length === 0 && this.queue_waiting.length === 0) {
this.emit('complete')
}
}
private startDownloadItem(item: DownloadOptionsID): DownloadItem {
const download = new DownloadItem(item)
download.on('error', (err) => {
this.emit('error', err)
this.checkQueue()
})
download.on('chunk', (_) => {
this.emit('update', this)
})
download.on('done', (d) => {
this.emit('done', d)
this.emit(`done_${d.options.id}`, d)
this.checkQueue()
})
return download
}
}
export interface DownloadItemEvents {
chunk: (item: DownloadItem) => void
done: (item: DownloadItem) => void
error: (message: string) => void
}
export declare interface DownloadItem {
on<U extends keyof DownloadItemEvents>(
event: U,
listener: DownloadItemEvents[U]
): this
once<U extends keyof DownloadItemEvents>(
event: U,
listener: DownloadItemEvents[U]
): this
emit<U extends keyof DownloadItemEvents>(
event: U,
...args: Parameters<DownloadItemEvents[U]>
): boolean
}
export class DownloadItem extends EventEmitter {
options: DownloadOptionsID
downloaded: number = 0
contentlength?: number
content = new Uint8Array(0)
completed = false
errored = false
constructor(options: DownloadOptionsID) {
super()
this.options = options
this.startDownload()
.then(() => {
this.completed = true
this.emit('done', this)
})
.catch((err) => {
this.errored = true
this.emit('error', err)
})
}
contentString(): string {
if (this.options.file || !this.completed) {
return ''
}
const decoder = new TextDecoder('utf-8')
return decoder.decode(this.content)
}
private updateContentLength(res: Response) {
const length = parseInt(res.headers.get('Content-Length') ?? '')
if (length && length > 0) {
this.contentlength = length
}
}
private async startDownload() {
try {
const res = await fetch(this.options.url)
if (res.status !== 200) {
throw `HTTP status code: ${res.status} ${res.statusText}`
}
if (!res.body) {
throw `HTTP response has no body`
}
this.updateContentLength(res)
this.emit('chunk', this)
if (this.options.file) {
await Deno.mkdir(this.options.file.fullpath, {
recursive: true,
})
const file = await Deno.open(this.options.file.fullfilepath, {
create: true,
write: true,
})
for await (const chunk of res.body) {
await file.write(chunk)
this.downloaded += chunk.length
this.emit('chunk', this)
}
} else {
const chunks = []
for await (const chunk of res.body) {
chunks.push(chunk)
this.downloaded += chunk.length
this.emit('chunk', this)
}
this.content = new Uint8Array(this.downloaded)
let offset = 0
for (const chunk of chunks) {
this.content.set(chunk, offset)
offset += chunk.length
}
}
} catch (e) {
throw e
}
}
}

108
src/main.ts Normal file
View File

@@ -0,0 +1,108 @@
import { parseArgs } from 'jsr:@std/cli/parse-args'
import { downloadSong, fetchPlaylist } from './web.ts'
import { DownloadOptions, DownloadQueue } from './download.ts'
import { UI } from './ui.ts'
import { getFileDetails } from './util.ts'
// parse args
const flags = parseArgs(Deno.args, {
alias: {
url: ['u'],
output: ['o'],
queuesize: ['qs'],
help: ['h'],
},
string: ['url', 'output', 'queuesize'],
boolean: ['help'],
default: {
url: '',
output: '.',
queuesize: '5',
},
})
if (!flags.url && flags._.length > 0) {
flags.url = flags._[0].toString()
}
const downloadQueueSize = parseInt(flags.queuesize)
if (!downloadQueueSize || downloadQueueSize < 1) {
console.log(`Unable to parse parse Queue Size: "${flags.queuesize}"`)
Deno.exit()
}
function printHelp() {
console.log(`deno
--allow-net --allow-write --allow-read main.ts
[--url] <url>
[--output <downloadPath>]
[--queuesize <int>]
[--help]
parameters:
[--url] <url>
alis: -u
required: true
url to download
--output <downloadPath>
alias: -o
default: "."
output path
--queuesize <queueSize>
alias: -qs
default: 5
number of files to download simultaneously
--help
alias: -h
print help message
`)
}
async function main() {
const downloadQueue = new DownloadQueue(5)
const ui = new UI(downloadQueue)
// load all song details
const playlistUrl = new URL(flags.url)
const { playlist, images } = await fetchPlaylist(
ui,
downloadQueue,
playlistUrl
)
const albumName = playlist[0].album
const imageDownloadOptions: DownloadOptions[] = images.map(
(imageUrl, i) => {
const imagename = decodeURIComponent(
imageUrl.pathname.split('/').pop() ?? `image_${i}`
)
const file = getFileDetails(flags.output, albumName, imagename)
return {
url: imageUrl,
name: `Image - ${file.file}`,
file,
}
}
)
downloadQueue.downloadBulk(...imageDownloadOptions)
for (const song of playlist) {
downloadSong(ui, downloadQueue, song, flags.output)
}
await downloadQueue.complete()
ui.printLogs()
}
if (!flags.url) {
console.log('Missing URL\n\n')
printHelp()
} else if (flags.help) {
printHelp()
} else {
main().catch((e) => console.error(e))
}

120
src/ui.ts Normal file
View File

@@ -0,0 +1,120 @@
import { DownloadQueue } from './download.ts'
import { getHeight, getWidth } from 'https://deno.land/x/terminal_size/mod.ts'
import { humanizeBytes, round } from './util.ts'
import { stdout } from 'node:process'
interface LogItem {
date: Date
message: string
}
export class UI {
private downloads: DownloadQueue
private log_history: LogItem[] = []
constructor(downloads: DownloadQueue) {
this.downloads = downloads
this.printDownloads()
downloads.on('update', (_) => this.printDownloads())
}
// deno-lint-ignore no-explicit-any
log(...message: any[]) {
const msg = message.map((m) => m.toString()).join(' ')
const date = new Date()
this.log_history.push({ date, message: msg })
this.printDownloads()
}
// deno-lint-ignore no-explicit-any
logExit(...message: any[]) {
this.log(...message)
this.printLogs()
Deno.exit()
}
printLogs() {
for (const logItem of this.log_history) {
const dateStr = logItem.date.toISOString()
console.log(`${dateStr} ${logItem.message}`)
}
}
private printDownloads() {
const terminalWidth = getWidth()
const terminaHeight = getHeight()
// console.log(terminalWidth, terminaHeight)
// process.exit()
const status = this.downloads.status()
let messageLines: string[] = []
// active queue
for (const item of status.active) {
const downloaded = humanizeBytes(item.downloaded)
const total = item.size ? humanizeBytes(item.size) : '???'
const percent = item.size
? round((item.downloaded / item.size) * 100, 0)
: '??'
{
let nameLine = `${item.name}`
const nameLineEnd = ` ${downloaded} / ${total} (${percent}%)`
const nameLineSpaces =
terminalWidth - (nameLine.length + nameLineEnd.length)
if (nameLineSpaces >= 0) {
const spaces = new Array(nameLineSpaces).fill(' ').join('')
nameLine = nameLine + spaces + nameLineEnd
} else {
nameLine =
nameLine.slice(0, nameLine.length + nameLineSpaces) +
nameLineEnd
}
messageLines.push(nameLine)
}
messageLines.push(` from: ${item.url.pathname}`)
messageLines.push(
` to: ${item.file ? item.file.relpath : 'Memory Buffer'}`
)
}
// other details
messageLines.push(`___`)
messageLines.push(
`Active/Waiting/Completed = ${status.active.length}/${status.waiting}/${status.completed}`
)
// trim away excess lines
const messageStartIndex = messageLines.length - terminaHeight
if (messageStartIndex >= 0) {
messageLines = messageLines.slice(messageStartIndex)
} else {
messageLines = [
...new Array(-messageStartIndex).fill(''),
...messageLines,
]
}
// trim excess line widths
messageLines = messageLines.map((line) => {
if (line.length >= terminalWidth) {
return line.slice(0, terminalWidth)
} else {
const spaces = new Array(terminalWidth - line.length)
.fill(' ')
.join('')
return line + spaces
}
// return line
})
// clear screen, go to top left
// stdout.write("\x1b[2J")
stdout.write(`\x1b[0;0H`)
// print lines
stdout.write(messageLines.join('\n'))
}
}

195
src/util.ts Normal file
View File

@@ -0,0 +1,195 @@
import { resolve, join } from 'node:path'
const REGEX_UNSAFEFORFILE = /[\\\/:*?'"<>|*!?]|^null$|^\./gi
/** example
* {
* path: 'foldername',
* fullpath: '/path/to/foldername',
* location: '/path/to',
*
* file: 'filename',
* ext: 'ext',
* fullfile: 'filename.ext
* relpath: 'foldername/filename.ext',
* fullfilepath: '/path/to/foldername/filename.ext',
* }
*/
export interface DownloadFolderDetails {
path: string
fullpath: string
location: string
}
export interface DownloadFileDetails extends DownloadFolderDetails {
file: string
ext: string
fullfile: string
relpath: string
fullfilepath: string
}
export interface PlaylistSongData {
url: URL
album: string
name: string
track: number
cd: number
}
export function songFoldername(
song: Partial<PlaylistSongData> | string
): string {
if (typeof song != 'string') {
song = song.album ?? ''
}
const albumname = song.replace(REGEX_UNSAFEFORFILE, '')
return albumname
}
export function songFilename(
song: Partial<PlaylistSongData>
): [filename: string, extension: string] {
if (!song.name) {
return ['', '']
}
const songname = song.name.replace(REGEX_UNSAFEFORFILE, '')
const tracknum = song.track ?? -1
const cdnum = song.cd ?? -1
const cd = cdnum >= 0 ? cdnum + '.' : ''
const track = tracknum >= 0 ? tracknum + '.' : ''
const separator = cdnum >= 0 || tracknum >= 0 ? ' ' : ''
return [`${cd}${track}${separator}${songname}`, 'flac']
/*
for example
song = {
name: 'song',
track: 1,
cd: 1,
}
then filename = '1.1. song'
song = {
name: 'song',
track: 1,
cd: -1,
}
then filename = '1. song'
song = {
name: 'song',
track: -1,
cd: -1,
}
then filename = 'song'
*/
}
export function splitFilename(
filename: string
): [filename: string, extension: string] {
const spl = filename.split('.')
let ext = 'bin'
if (spl.length > 1) {
ext = spl.pop() ?? ext
}
return [spl.join('.'), ext]
}
export function getSongFileDetails(
location: string,
song: Partial<PlaylistSongData>
): DownloadFileDetails {
const path = songFoldername(song)
const [file, ext] = songFilename(song)
const fullfile = `${file}.${ext}`
const relpath = join(path, fullfile)
const fullpath = resolve(location, path)
const fullfilepath = resolve(fullpath, fullfile)
return {
path,
file,
ext,
fullfile,
relpath,
fullpath,
fullfilepath,
location,
}
}
export function getFileDetails(
location: string,
albumName: string,
filename: string
): DownloadFileDetails {
const path = songFoldername(albumName)
const [file, ext] = splitFilename(filename)
const fullfile = `${file}.${ext}`
const relpath = join(path, fullfile)
const fullpath = resolve(location, path)
const fullfilepath = resolve(fullpath, fullfile)
return {
path,
file,
ext,
fullfile,
relpath,
fullpath,
fullfilepath,
location,
}
}
const BYTE_MAGNITUDE = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
export function humanizeBytes(size: number, decimals: number = 2) {
let magnitude = 0
while (size > 1000) {
magnitude++
size /= 1000
}
size = round(size, decimals)
let magnitudeString = ''
if (magnitude < BYTE_MAGNITUDE.length) {
magnitudeString = BYTE_MAGNITUDE[magnitude]
} else {
const excessMagnitude = magnitude - BYTE_MAGNITUDE.length
magnitudeString = `${
BYTE_MAGNITUDE[BYTE_MAGNITUDE.length - 1]
}+${excessMagnitude}`
}
return `${size}${magnitudeString}`
}
export function round(number: number, decimals: number = 2) {
decimals = Math.max(0, Math.round(decimals))
const decimalsMagnitude = Math.pow(10, decimals)
return Math.round(number * decimalsMagnitude) / decimalsMagnitude
}
const CHARS =
'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.split('')
export function randomChar() {
return CHARS[Math.round(Math.random() * (CHARS.length - 1))]
}
export function randomChars(length: number) {
return new Array(length)
.fill('')
.map((_) => randomChar())
.join('')
}

241
src/web.ts Normal file
View File

@@ -0,0 +1,241 @@
import { exists } from 'jsr:@std/fs/exists'
import * as cheerio from 'https://esm.sh/cheerio?target=esnext'
import { DownloadQueue } from './download.ts'
import { UI } from './ui.ts'
import {
DownloadFileDetails,
getSongFileDetails,
PlaylistSongData,
} from './util.ts'
const REGEX_PLAYLISTURL =
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/?$/i
const REGEX_SONGURL =
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/[^\/]+\/?$/i
const REGEX_ALBUMTITLE = /^(.*?) MP3 - Download/i
export interface PlaylistWhole {
playlist: PlaylistSongData[]
images: URL[]
}
export async function fetchPlaylist(
ui: UI,
downloadQueue: DownloadQueue,
url: URL
): Promise<PlaylistWhole> {
if (!REGEX_PLAYLISTURL.test(url.toString())) {
throw `unaccepted url ${url}`
}
ui.log(`downloading: ${url}`)
// load the playlist page's dom
const req = await downloadQueue.downloadPromise({
name: 'Playlist HTML',
url,
})
const text = req.contentString()
const $ = cheerio.load(text)
// get the album name from the page title
const title =
$.extract({
title: 'title',
}).title ?? ''
const titleMatch = REGEX_ALBUMTITLE.exec(title)
if (titleMatch === null) {
throw `unable to grab album name from ${title}`
}
const albumName = titleMatch[1]
ui.log(` title: ${albumName}`)
// get the album images from the page
const images =
$.extract({
images: [
{
selector: '.albumImage a',
value: 'href',
},
],
}).images ?? []
const imageUrls = images.map((img) => new URL(img))
ui.log(` images: ${images.length}`)
// parse all rows in playlist
const columns = getPlaylistTableHeaders($)
const rows = getPlaylistRows($, columns, albumName)
ui.log(` songs: ${rows.length}\n`)
return { playlist: rows, images: imageUrls }
}
interface PlaylistTableColumns {
name: number
track: number
cd: number
}
function getPlaylistTableHeaders($: cheerio.CheerioAPI): PlaylistTableColumns {
// get table header row
const header = $('#songlist tr#songlist_header')
const headerCells = header.extract({
cells: ['th'],
})
// get the index for specific columns
const indexes = headerCells.cells.reduce(
(p, c, i) => {
// check the string content of the current cell
switch (c.toLocaleLowerCase()) {
case 'cd':
p.cd = i
break
case '#':
p.track = i
break
case 'song name':
p.name = i
}
return p
},
// default values
{
name: -1,
track: -1,
cd: -1,
}
)
if (indexes.name == -1) {
throw 'unable to find song title column'
}
return indexes
}
function getPlaylistRows(
$: cheerio.CheerioAPI,
columns: PlaylistTableColumns,
albumName: string
): PlaylistSongData[] {
const rows = $('#songlist tr:not(#songlist_header):not(#songlist_footer)')
const rowsData: PlaylistSongData[] = []
// loop through each song in table
rows.each((_, rowEl) => {
const row = cheerio.load(rowEl)
// prase values from row
const urlStr =
row.extract({
url: {
selector: 'td.playlistDownloadSong a',
value: 'href',
},
}).url ?? ''
const rowData: PlaylistSongData = {
url: newKHInsiderURL(urlStr),
album: albumName,
name: '',
track: -1,
cd: -1,
}
const rowCells = row.extract({
cells: ['td'],
}).cells
rowData.name = rowCells[columns.name]
if (!rowData.name) {
throw `unable to grab song name from ${albumName} in row ${columns.name} - ${rowCells}`
}
if (columns.track >= 0) {
rowData.track = parseInt(rowCells[columns.track] ?? '-1')
if (isNaN(rowData.track)) {
rowData.track = -1
}
}
if (columns.cd >= 0) {
rowData.cd = parseInt(rowCells[columns.cd] ?? '-1')
if (isNaN(rowData.cd)) {
rowData.cd = -1
}
}
rowsData.push(rowData)
})
return rowsData
}
function newKHInsiderURL(url: string) {
if (!/^http/i.test(url.toString())) {
return new URL('https://downloads.khinsider.com' + url)
}
return new URL(url)
}
export async function downloadSong(
ui: UI,
downloadQueue: DownloadQueue,
song: PlaylistSongData,
location: string
) {
// get full url
if (!REGEX_SONGURL.test(song.url.toString())) {
throw `unaccepted url ${song.url}`
}
const file = getSongFileDetails(location, song)
// skip file if it exists
if (await exists(file.fullfilepath)) {
ui.log(`skipping file already exists ${file.fullfilepath}`)
return
}
// get file url
const flacURL = await getFileURL(downloadQueue, song, file)
ui.log(`downloading ${file.relpath}`)
ui.log(` from ${flacURL}`)
// download the file
downloadQueue.download({
url: flacURL,
name: `File - ${file.file}`,
file,
})
}
async function getFileURL(
downloadQueue: DownloadQueue,
song: PlaylistSongData,
file: DownloadFileDetails
) {
// load download page
const resp = await downloadQueue.downloadPromise({
url: song.url,
name: `DL Page - ${file.file}`,
})
const text = resp.contentString()
const $ = cheerio.load(text)
// extract the flac download link
const flacUrlStr = $.extract({
url: {
selector: '#pageContent a[href*="flac"]',
value: 'href',
},
}).url
if (!flacUrlStr) {
throw `can't find download link for ${song.url}`
}
return new URL(flacUrlStr)
}