created a download queue, added full screen status viewing
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
{
|
||||
"tabWidth": 2,
|
||||
"tabWidth": 4,
|
||||
"useTabs": false,
|
||||
"semi": false,
|
||||
"singleQuote": true,
|
||||
"objectWrap": "collapse"
|
||||
"singleQuote": true
|
||||
}
|
||||
|
||||
44
deno.lock
generated
44
deno.lock
generated
@@ -31,6 +31,7 @@
|
||||
}
|
||||
},
|
||||
"redirects": {
|
||||
"https://deno.land/x/terminal_size/mod.ts": "https://deno.land/x/terminal_size@0.1.0/mod.ts",
|
||||
"https://esm.sh/@types/boolbase@~1.0.3/index.d.ts": "https://esm.sh/@types/boolbase@1.0.3/index.d.ts",
|
||||
"https://esm.sh/@types/safer-buffer@~2.1.3/index.d.ts": "https://esm.sh/@types/safer-buffer@2.1.3/index.d.ts",
|
||||
"https://esm.sh/boolbase@^1.0.0?target=deno": "https://esm.sh/boolbase@1.0.0?target=deno",
|
||||
@@ -89,6 +90,18 @@
|
||||
"https://esm.sh/whatwg-mimetype@^4.0.0?target=denonext": "https://esm.sh/whatwg-mimetype@4.0.0?target=denonext"
|
||||
},
|
||||
"remote": {
|
||||
"https://deno.land/std@0.122.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58",
|
||||
"https://deno.land/std@0.122.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac",
|
||||
"https://deno.land/std@0.122.0/fmt/colors.ts": "8368ddf2d48dfe413ffd04cdbb7ae6a1009cf0dccc9c7ff1d76259d9c61a0621",
|
||||
"https://deno.land/std@0.122.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853",
|
||||
"https://deno.land/std@0.122.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4",
|
||||
"https://deno.land/std@0.122.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b",
|
||||
"https://deno.land/std@0.122.0/path/common.ts": "f41a38a0719a1e85aa11c6ba3bea5e37c15dd009d705bd8873f94c833568cbc4",
|
||||
"https://deno.land/std@0.122.0/path/glob.ts": "7bf2349e818e332a830f3d8874c3f45dd7580b6c742ed50dbf6282d84ab18405",
|
||||
"https://deno.land/std@0.122.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12",
|
||||
"https://deno.land/std@0.122.0/path/posix.ts": "34349174b9cd121625a2810837a82dd8b986bbaaad5ade690d1de75bbb4555b2",
|
||||
"https://deno.land/std@0.122.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c",
|
||||
"https://deno.land/std@0.122.0/path/win32.ts": "11549e8c6df8307a8efcfa47ad7b2a75da743eac7d4c89c9723a944661c8bd2e",
|
||||
"https://deno.land/std@0.177.1/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462",
|
||||
"https://deno.land/std@0.177.1/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3",
|
||||
"https://deno.land/std@0.177.1/async/abortable.ts": "73acfb3ed7261ce0d930dbe89e43db8d34e017b063cf0eaa7d215477bf53442e",
|
||||
@@ -377,6 +390,37 @@
|
||||
"https://deno.land/std@0.177.1/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea",
|
||||
"https://deno.land/std@0.177.1/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7",
|
||||
"https://deno.land/std@0.177.1/testing/asserts.ts": "984ab0bfb3faeed92ffaa3a6b06536c66811185328c5dd146257c702c41b01ab",
|
||||
"https://deno.land/std@0.97.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58",
|
||||
"https://deno.land/std@0.97.0/_util/os.ts": "e282950a0eaa96760c0cf11e7463e66babd15ec9157d4c9ed49cc0925686f6a7",
|
||||
"https://deno.land/std@0.97.0/encoding/base64.ts": "eecae390f1f1d1cae6f6c6d732ede5276bf4b9cd29b1d281678c054dc5cc009e",
|
||||
"https://deno.land/std@0.97.0/encoding/hex.ts": "f952e0727bddb3b2fd2e6889d104eacbd62e92091f540ebd6459317a61932d9b",
|
||||
"https://deno.land/std@0.97.0/fs/_util.ts": "f2ce811350236ea8c28450ed822a5f42a0892316515b1cd61321dec13569c56b",
|
||||
"https://deno.land/std@0.97.0/fs/ensure_dir.ts": "b7c103dc41a3d1dbbb522bf183c519c37065fdc234831a4a0f7d671b1ed5fea7",
|
||||
"https://deno.land/std@0.97.0/fs/exists.ts": "b0d2e31654819cc2a8d37df45d6b14686c0cc1d802e9ff09e902a63e98b85a00",
|
||||
"https://deno.land/std@0.97.0/hash/_wasm/hash.ts": "cb6ad1ab429f8ac9d6eae48f3286e08236d662e1a2e5cfd681ba1c0f17375895",
|
||||
"https://deno.land/std@0.97.0/hash/_wasm/wasm.js": "94b1b997ae6fb4e6d2156bcea8f79cfcd1e512a91252b08800a92071e5e84e1a",
|
||||
"https://deno.land/std@0.97.0/hash/hasher.ts": "57a9ec05dd48a9eceed319ac53463d9873490feea3832d58679df6eec51c176b",
|
||||
"https://deno.land/std@0.97.0/hash/mod.ts": "5d032bd34186cda2f8d17fc122d621430953a6030d4b3f11172004715e3e2441",
|
||||
"https://deno.land/std@0.97.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853",
|
||||
"https://deno.land/std@0.97.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4",
|
||||
"https://deno.land/std@0.97.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b",
|
||||
"https://deno.land/std@0.97.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a",
|
||||
"https://deno.land/std@0.97.0/path/glob.ts": "314ad9ff263b895795208cdd4d5e35a44618ca3c6dd155e226fb15d065008652",
|
||||
"https://deno.land/std@0.97.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12",
|
||||
"https://deno.land/std@0.97.0/path/posix.ts": "f56c3c99feb47f30a40ce9d252ef6f00296fa7c0fcb6dd81211bdb3b8b99ca3b",
|
||||
"https://deno.land/std@0.97.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c",
|
||||
"https://deno.land/std@0.97.0/path/win32.ts": "77f7b3604e0de40f3a7c698e8a79e7f601dc187035a1c21cb1e596666ce112f8",
|
||||
"https://deno.land/x/cache@0.2.13/cache.ts": "4005aad54fb9aac9ff02526ffa798032e57f2d7966905fdeb7949263b1c95f2f",
|
||||
"https://deno.land/x/cache@0.2.13/deps.ts": "6f14e76a1a09f329e3f3830c6e72bd10b53a89a75769d5ea886e5d8603e503e6",
|
||||
"https://deno.land/x/cache@0.2.13/directories.ts": "ef48531cab3f827252e248596d15cede0de179a2fb15392ae24cf8034519994f",
|
||||
"https://deno.land/x/cache@0.2.13/file.ts": "5abe7d80c6ac594c98e66eb4262962139f48cd9c49dbe2a77e9608760508a09a",
|
||||
"https://deno.land/x/cache@0.2.13/file_fetcher.ts": "5c793cc83a5b9377679ec313b2a2321e51bf7ed15380fa82d387f1cdef3b924f",
|
||||
"https://deno.land/x/cache@0.2.13/helpers.ts": "d1545d6432277b7a0b5ea254d1c51d572b6452a8eadd9faa7ad9c5586a1725c4",
|
||||
"https://deno.land/x/cache@0.2.13/mod.ts": "3188250d3a013ef6c9eb060e5284cf729083af7944a29e60bb3d8597dd20ebcd",
|
||||
"https://deno.land/x/plug@0.5.1/deps.ts": "0f53866f60dc4f89bbc3be9d096f7501f2068a51923db220f43f0ad284b6b892",
|
||||
"https://deno.land/x/plug@0.5.1/plug.ts": "620f9454df049228ba715b215582200a344f33f37605d5b4dfce5bf89c7e9815",
|
||||
"https://deno.land/x/terminal_size@0.1.0/bindings/bindings.ts": "712ba91bfc7c93d5bbad2e6c996ccc062d9a12ac3c0fa258ed068f666ce43d3b",
|
||||
"https://deno.land/x/terminal_size@0.1.0/mod.ts": "6ab2530394db2675f6564888e14bbafa19bc887976786149e4f215b8e6d3038f",
|
||||
"https://esm.sh/boolbase@1.0.0/deno/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb",
|
||||
"https://esm.sh/boolbase@1.0.0/denonext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb",
|
||||
"https://esm.sh/boolbase@1.0.0/esnext/boolbase.mjs": "70e9521b9532b5e4dc0c807422529b15b4452663dbdb70dff9c7b65d0ff2e3cb",
|
||||
|
||||
333
main.ts
333
main.ts
@@ -1,333 +0,0 @@
|
||||
import path from 'node:path'
|
||||
import { Buffer } from 'node:buffer'
|
||||
import { parseArgs } from "jsr:@std/cli/parse-args"
|
||||
import { exists } from "jsr:@std/fs/exists";
|
||||
import * as cheerio from 'https://esm.sh/cheerio?target=esnext'
|
||||
|
||||
const REGEX_PLAYLISTURL =
|
||||
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/?$/i
|
||||
const REGEX_SONGURL =
|
||||
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/[^\/]+\/?$/i
|
||||
const REGEX_ALBUMTITLE = /^(.*?) MP3 - Download/i
|
||||
const REGEX_UNSAFEFORFILE = /[^a-z0-9\-_=+,.()\[\]{} ]/gi
|
||||
|
||||
// parse args
|
||||
const flags = parseArgs(Deno.args, {
|
||||
alias: {
|
||||
"url": ["u"],
|
||||
"output": ["o"],
|
||||
"sync": ["s"],
|
||||
"help": ["h"],
|
||||
},
|
||||
string: ["url", "output"],
|
||||
boolean: ["sync", "help"],
|
||||
default: {
|
||||
"url": "",
|
||||
"output": ".",
|
||||
"sync": false
|
||||
}
|
||||
})
|
||||
|
||||
if (!flags.url && flags._.length > 0) {
|
||||
flags.url = flags._[0].toString()
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`deno
|
||||
--allow-net --allow-write --allow-read main.ts
|
||||
[--url] <url>
|
||||
[--output <downloadPath>]
|
||||
[--sync]
|
||||
[--help]
|
||||
|
||||
parameters:
|
||||
--url -u (default)
|
||||
url to download
|
||||
|
||||
--output -o
|
||||
default: "."
|
||||
output path
|
||||
|
||||
--sync -s
|
||||
download files one at a time
|
||||
|
||||
--help -h
|
||||
print help message
|
||||
`)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// load all song details
|
||||
const playlist = await fetchPlaylist(flags.url)
|
||||
|
||||
// if sync, download one at a time
|
||||
if (flags.sync) {
|
||||
console.log('downloading one at a time\n')
|
||||
for (const song of playlist) {
|
||||
await downloadSong(song, flags.output || '.')
|
||||
}
|
||||
} else {
|
||||
console.log('downloading all at once\n')
|
||||
await Promise.all(
|
||||
playlist.map((song) => downloadSong(song, flags.output || '.'))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchPlaylist(url: string): Promise<PlaylistSongData[]> {
|
||||
if (!REGEX_PLAYLISTURL.test(url)) {
|
||||
throw `unaccepted url ${url}`
|
||||
}
|
||||
|
||||
console.log(`downloading: ${url}`)
|
||||
|
||||
// load the playlist page's dom
|
||||
const req = await fetch(url)
|
||||
const text = await req.text()
|
||||
const $ = cheerio.load(text)
|
||||
|
||||
// get the album name from the page title
|
||||
const title =
|
||||
$.extract({
|
||||
title: 'title',
|
||||
}).title ?? ''
|
||||
const titleMatch = REGEX_ALBUMTITLE.exec(title)
|
||||
if (titleMatch === null) {
|
||||
throw `unable to grab album name from ${title}`
|
||||
}
|
||||
const albumName = titleMatch[1]
|
||||
console.log(` title: ${albumName}`)
|
||||
|
||||
// parse all rows in playlist
|
||||
const columns = getPlaylistTableHeaders($)
|
||||
const rows = getPlaylistRows($, columns, albumName)
|
||||
console.log(` songs: ${rows.length}\n`)
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
interface PlaylistTableColumns {
|
||||
name: number
|
||||
track: number
|
||||
cd: number
|
||||
}
|
||||
|
||||
interface PlaylistSongData {
|
||||
url: string
|
||||
album: string
|
||||
name: string
|
||||
track: number
|
||||
cd: number
|
||||
}
|
||||
|
||||
function getPlaylistTableHeaders($: cheerio.CheerioAPI): PlaylistTableColumns {
|
||||
// get table header row
|
||||
const header = $('#songlist tr#songlist_header')
|
||||
const headerCells = header.extract({
|
||||
cells: ['th'],
|
||||
})
|
||||
|
||||
// get the index for specific columns
|
||||
const indexes = headerCells.cells.reduce(
|
||||
(p, c, i) => {
|
||||
// check the string content of the current cell
|
||||
switch (c.toLocaleLowerCase()) {
|
||||
case 'cd':
|
||||
p.cd = i
|
||||
break
|
||||
case '#':
|
||||
p.track = i
|
||||
break
|
||||
case 'song name':
|
||||
p.name = i
|
||||
}
|
||||
return p
|
||||
},
|
||||
// default values
|
||||
{
|
||||
name: -1,
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
)
|
||||
|
||||
if (indexes.name == -1) {
|
||||
throw 'unable to find song title column'
|
||||
}
|
||||
|
||||
return indexes
|
||||
}
|
||||
|
||||
function getPlaylistRows(
|
||||
$: cheerio.CheerioAPI,
|
||||
columns: PlaylistTableColumns,
|
||||
albumName: string
|
||||
): PlaylistSongData[] {
|
||||
const rows = $('#songlist tr:not(#songlist_header):not(#songlist_footer)')
|
||||
const rowsData: PlaylistSongData[] = []
|
||||
|
||||
// loop through each song in table
|
||||
rows.each((_, rowEl) => {
|
||||
const row = cheerio.load(rowEl)
|
||||
const rowData: PlaylistSongData = {
|
||||
url: '',
|
||||
album: albumName,
|
||||
name: '',
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
|
||||
// prase values from row
|
||||
rowData.url =
|
||||
row.extract({
|
||||
url: {
|
||||
selector: 'td.playlistDownloadSong a',
|
||||
value: 'href',
|
||||
},
|
||||
}).url ?? ''
|
||||
|
||||
const rowCells = row.extract({
|
||||
cells: ['td'],
|
||||
}).cells
|
||||
|
||||
rowData.name = rowCells[columns.name]
|
||||
if (!rowData.name) {
|
||||
throw `unable to grab song name from ${albumName} in row ${columns.name} - ${rowCells}`
|
||||
}
|
||||
|
||||
if (columns.track >= 0) {
|
||||
rowData.track = parseInt(rowCells[columns.track] ?? '-1')
|
||||
if (isNaN(rowData.track)) {
|
||||
rowData.track = -1
|
||||
}
|
||||
}
|
||||
|
||||
if (columns.cd >= 0) {
|
||||
rowData.cd = parseInt(rowCells[columns.cd] ?? '-1')
|
||||
if (isNaN(rowData.cd)) {
|
||||
rowData.cd = -1
|
||||
}
|
||||
}
|
||||
|
||||
rowsData.push(rowData)
|
||||
})
|
||||
|
||||
return rowsData
|
||||
}
|
||||
|
||||
async function downloadSong(song: PlaylistSongData, location: string) {
|
||||
// get full url
|
||||
let url = song.url
|
||||
if (!/^http/i.test(url)) {
|
||||
url = 'https://downloads.khinsider.com' + url
|
||||
}
|
||||
if (!REGEX_SONGURL.test(url)) {
|
||||
throw `unaccepted url ${url}`
|
||||
}
|
||||
|
||||
// load download page
|
||||
const resp = await fetch(url)
|
||||
const text = await resp.text()
|
||||
const $ = cheerio.load(text)
|
||||
|
||||
// extract the flac download link
|
||||
const flacUrl = $.extract({
|
||||
url: {
|
||||
selector: '#pageContent a[href*="flac"]',
|
||||
value: 'href',
|
||||
},
|
||||
}).url
|
||||
|
||||
if (!flacUrl) {
|
||||
throw `can't find download link for ${url}`
|
||||
}
|
||||
|
||||
// get the file and path to save the files
|
||||
const { pathname, fullpathname } = pathFor(location, song)
|
||||
|
||||
// ensure folder exists
|
||||
if (!await exists(pathname)) {
|
||||
await Deno.mkdir(pathname)
|
||||
}
|
||||
|
||||
// skip file if it exists
|
||||
if (await exists(fullpathname)) {
|
||||
console.log(`skipping file already exists ${fullpathname}`)
|
||||
}
|
||||
|
||||
console.log(`downloading ${fullpathname}`)
|
||||
console.log(` from ${flacUrl}`)
|
||||
|
||||
// download the file
|
||||
const songresp = await fetch(flacUrl)
|
||||
const songblob = await songresp.arrayBuffer()
|
||||
return Deno.writeFile(fullpathname, toBuffer(songblob))
|
||||
}
|
||||
|
||||
interface SongPath {
|
||||
pathname: string
|
||||
fullpathname: string
|
||||
}
|
||||
|
||||
function pathFor(location: string, song: PlaylistSongData): SongPath {
|
||||
// clean strings for file paths
|
||||
const albumname = song.album.replace(REGEX_UNSAFEFORFILE, '')
|
||||
const songname = song.name.replace(REGEX_UNSAFEFORFILE, '')
|
||||
|
||||
const cd = song.cd >= 0 ? song.cd + '.' : ''
|
||||
const track = song.track >= 0 ? song.track + '.' : ''
|
||||
const separator = song.cd >= 0 || song.track >= 0 ? ' ' : ''
|
||||
const filename = `${cd}${track}${separator}${songname}.flac`
|
||||
/*
|
||||
for example
|
||||
|
||||
song = {
|
||||
songname: 'song',
|
||||
track: 1,
|
||||
cd: 1,
|
||||
}
|
||||
then filename = '1.1. song'
|
||||
|
||||
song = {
|
||||
songname: 'song',
|
||||
track: 1,
|
||||
cd: -1,
|
||||
}
|
||||
then filename = '1. song'
|
||||
|
||||
song = {
|
||||
songname: 'song',
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
then filename = 'song'
|
||||
*/
|
||||
|
||||
const pathname = path.resolve(location, albumname)
|
||||
const fullpathname = path.resolve(pathname, filename)
|
||||
|
||||
return {
|
||||
pathname,
|
||||
fullpathname,
|
||||
}
|
||||
}
|
||||
|
||||
// convert ArrayBuffer to Buffer<ArrayBuffer>
|
||||
function toBuffer(arrayBuffer: ArrayBuffer): Buffer<ArrayBuffer> {
|
||||
const buffer = Buffer.alloc(arrayBuffer.byteLength)
|
||||
const view = new Uint8Array(arrayBuffer)
|
||||
for (let i = 0; i < buffer.length; ++i) {
|
||||
buffer[i] = view[i]
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
if (!flags.url) {
|
||||
console.log('Missing URL\n\n')
|
||||
printHelp()
|
||||
} else if (flags.help) {
|
||||
printHelp()
|
||||
} else {
|
||||
main().catch((e) => console.error(e))
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{
|
||||
"main": "src/main.ts",
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.5.2"
|
||||
}
|
||||
|
||||
302
src/download.ts
Normal file
302
src/download.ts
Normal file
@@ -0,0 +1,302 @@
|
||||
import EventEmitter from 'node:events'
|
||||
import { DownloadFileDetails, randomChars } from './util.ts'
|
||||
|
||||
export interface DownloadOptions {
|
||||
name: string
|
||||
url: URL
|
||||
file?: DownloadFileDetails
|
||||
}
|
||||
|
||||
export interface DownloadOptionsID extends DownloadOptions {
|
||||
id: string
|
||||
}
|
||||
|
||||
export interface DownloadItemStatus extends DownloadOptions {
|
||||
downloaded: number
|
||||
size?: number
|
||||
}
|
||||
|
||||
export interface DownloadStatus {
|
||||
waiting: number
|
||||
completed: number
|
||||
active: DownloadItemStatus[]
|
||||
}
|
||||
|
||||
export interface DownloadQueueEvents {
|
||||
update: (item: DownloadQueue) => void
|
||||
done: (item: DownloadItem) => void
|
||||
complete: () => void
|
||||
error: (message: string) => void
|
||||
}
|
||||
|
||||
export declare interface DownloadQueue {
|
||||
on<U extends keyof DownloadQueueEvents>(
|
||||
event: U,
|
||||
listener: DownloadQueueEvents[U]
|
||||
): this
|
||||
on(event: `done_${string}`, listener: DownloadQueueEvents['done']): this
|
||||
|
||||
once<U extends keyof DownloadQueueEvents>(
|
||||
event: U,
|
||||
listener: DownloadQueueEvents[U]
|
||||
): this
|
||||
once(event: `done_${string}`, listener: DownloadQueueEvents['done']): this
|
||||
|
||||
emit<U extends keyof DownloadQueueEvents>(
|
||||
event: U,
|
||||
...args: Parameters<DownloadQueueEvents[U]>
|
||||
): boolean
|
||||
emit(
|
||||
event: `done_${string}`,
|
||||
...args: Parameters<DownloadQueueEvents['done']>
|
||||
): this
|
||||
}
|
||||
|
||||
export class DownloadQueue extends EventEmitter {
|
||||
private queue_waiting: DownloadOptionsID[] = []
|
||||
private queue_active: DownloadItem[] = []
|
||||
private queue_history: DownloadItem[] = []
|
||||
private queue_size = 1
|
||||
private checking_queue = false
|
||||
|
||||
constructor(queue_size: number) {
|
||||
super()
|
||||
if (queue_size > 1) {
|
||||
this.queue_size = queue_size
|
||||
}
|
||||
}
|
||||
|
||||
downloadBulk(...options: DownloadOptions[]): string[] {
|
||||
const ids: string[] = []
|
||||
this.queue_waiting.push(
|
||||
...options.map((opt) => {
|
||||
const id = this.generateID()
|
||||
ids.push(id)
|
||||
return { ...opt, id }
|
||||
})
|
||||
)
|
||||
this.checkQueue()
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
download(options: DownloadOptions): string {
|
||||
const id = this.generateID()
|
||||
this.queue_waiting.push({ ...options, id })
|
||||
this.checkQueue()
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
downloadPromise(options: DownloadOptions): Promise<DownloadItem> {
|
||||
const id = this.download(options)
|
||||
return new Promise((resolve) => {
|
||||
this.once(`done_${id}`, (item) => {
|
||||
resolve(item)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
complete(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
this.on('complete', () => resolve())
|
||||
})
|
||||
}
|
||||
|
||||
private IDExists(id: string): boolean {
|
||||
const waitingIds = this.queue_waiting.map((item) => item.id)
|
||||
const queuedIds = [...this.queue_active, ...this.queue_history].map(
|
||||
(item) => item.options.id
|
||||
)
|
||||
return [...waitingIds, ...queuedIds].some((itemId) => itemId === id)
|
||||
}
|
||||
|
||||
private generateID(): string {
|
||||
let id = ''
|
||||
do {
|
||||
id = randomChars(16)
|
||||
} while (this.IDExists(id))
|
||||
return id
|
||||
}
|
||||
|
||||
status(): DownloadStatus {
|
||||
return {
|
||||
waiting: this.queue_waiting.length,
|
||||
completed: this.queue_history.length,
|
||||
active: this.queue_active.map((item) => ({
|
||||
...item.options,
|
||||
downloaded: item.downloaded,
|
||||
size: item.contentlength,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
private checkQueue() {
|
||||
if (this.checking_queue) {
|
||||
return
|
||||
}
|
||||
|
||||
this.checking_queue = true
|
||||
|
||||
// update active queue
|
||||
const queue_active: DownloadItem[] = []
|
||||
for (const item of this.queue_active) {
|
||||
if (item.completed) {
|
||||
this.queue_history.push(item)
|
||||
} else if (item.errored) {
|
||||
this.queue_waiting.push(item.options)
|
||||
} else {
|
||||
queue_active.push(item)
|
||||
}
|
||||
}
|
||||
this.queue_active = queue_active
|
||||
|
||||
// add to active queue
|
||||
while (
|
||||
this.queue_active.length < this.queue_size &&
|
||||
this.queue_waiting.length > 0
|
||||
) {
|
||||
const item = this.queue_waiting.shift()
|
||||
if (!item) {
|
||||
break
|
||||
}
|
||||
|
||||
this.queue_active.push(this.startDownloadItem(item))
|
||||
}
|
||||
|
||||
this.checking_queue = false
|
||||
this.emit('update', this)
|
||||
|
||||
if (this.queue_active.length === 0 && this.queue_waiting.length === 0) {
|
||||
this.emit('complete')
|
||||
}
|
||||
}
|
||||
|
||||
private startDownloadItem(item: DownloadOptionsID): DownloadItem {
|
||||
const download = new DownloadItem(item)
|
||||
download.on('error', (err) => {
|
||||
this.emit('error', err)
|
||||
this.checkQueue()
|
||||
})
|
||||
download.on('chunk', (_) => {
|
||||
this.emit('update', this)
|
||||
})
|
||||
download.on('done', (d) => {
|
||||
this.emit('done', d)
|
||||
this.emit(`done_${d.options.id}`, d)
|
||||
this.checkQueue()
|
||||
})
|
||||
return download
|
||||
}
|
||||
}
|
||||
|
||||
export interface DownloadItemEvents {
|
||||
chunk: (item: DownloadItem) => void
|
||||
done: (item: DownloadItem) => void
|
||||
error: (message: string) => void
|
||||
}
|
||||
|
||||
export declare interface DownloadItem {
|
||||
on<U extends keyof DownloadItemEvents>(
|
||||
event: U,
|
||||
listener: DownloadItemEvents[U]
|
||||
): this
|
||||
once<U extends keyof DownloadItemEvents>(
|
||||
event: U,
|
||||
listener: DownloadItemEvents[U]
|
||||
): this
|
||||
emit<U extends keyof DownloadItemEvents>(
|
||||
event: U,
|
||||
...args: Parameters<DownloadItemEvents[U]>
|
||||
): boolean
|
||||
}
|
||||
|
||||
export class DownloadItem extends EventEmitter {
|
||||
options: DownloadOptionsID
|
||||
downloaded: number = 0
|
||||
contentlength?: number
|
||||
content = new Uint8Array(0)
|
||||
|
||||
completed = false
|
||||
errored = false
|
||||
|
||||
constructor(options: DownloadOptionsID) {
|
||||
super()
|
||||
this.options = options
|
||||
this.startDownload()
|
||||
.then(() => {
|
||||
this.completed = true
|
||||
this.emit('done', this)
|
||||
})
|
||||
.catch((err) => {
|
||||
this.errored = true
|
||||
this.emit('error', err)
|
||||
})
|
||||
}
|
||||
|
||||
contentString(): string {
|
||||
if (this.options.file || !this.completed) {
|
||||
return ''
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder('utf-8')
|
||||
return decoder.decode(this.content)
|
||||
}
|
||||
|
||||
private updateContentLength(res: Response) {
|
||||
const length = parseInt(res.headers.get('Content-Length') ?? '')
|
||||
if (length && length > 0) {
|
||||
this.contentlength = length
|
||||
}
|
||||
}
|
||||
|
||||
private async startDownload() {
|
||||
try {
|
||||
const res = await fetch(this.options.url)
|
||||
if (res.status !== 200) {
|
||||
throw `HTTP status code: ${res.status} ${res.statusText}`
|
||||
}
|
||||
|
||||
if (!res.body) {
|
||||
throw `HTTP response has no body`
|
||||
}
|
||||
|
||||
this.updateContentLength(res)
|
||||
this.emit('chunk', this)
|
||||
|
||||
if (this.options.file) {
|
||||
await Deno.mkdir(this.options.file.fullpath, {
|
||||
recursive: true,
|
||||
})
|
||||
const file = await Deno.open(this.options.file.fullfilepath, {
|
||||
create: true,
|
||||
write: true,
|
||||
})
|
||||
|
||||
for await (const chunk of res.body) {
|
||||
await file.write(chunk)
|
||||
this.downloaded += chunk.length
|
||||
|
||||
this.emit('chunk', this)
|
||||
}
|
||||
} else {
|
||||
const chunks = []
|
||||
for await (const chunk of res.body) {
|
||||
chunks.push(chunk)
|
||||
this.downloaded += chunk.length
|
||||
|
||||
this.emit('chunk', this)
|
||||
}
|
||||
|
||||
this.content = new Uint8Array(this.downloaded)
|
||||
let offset = 0
|
||||
for (const chunk of chunks) {
|
||||
this.content.set(chunk, offset)
|
||||
offset += chunk.length
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
108
src/main.ts
Normal file
108
src/main.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
import { parseArgs } from 'jsr:@std/cli/parse-args'
|
||||
import { downloadSong, fetchPlaylist } from './web.ts'
|
||||
import { DownloadOptions, DownloadQueue } from './download.ts'
|
||||
import { UI } from './ui.ts'
|
||||
import { getFileDetails } from './util.ts'
|
||||
|
||||
// parse args
|
||||
const flags = parseArgs(Deno.args, {
|
||||
alias: {
|
||||
url: ['u'],
|
||||
output: ['o'],
|
||||
queuesize: ['qs'],
|
||||
help: ['h'],
|
||||
},
|
||||
string: ['url', 'output', 'queuesize'],
|
||||
boolean: ['help'],
|
||||
default: {
|
||||
url: '',
|
||||
output: '.',
|
||||
queuesize: '5',
|
||||
},
|
||||
})
|
||||
|
||||
if (!flags.url && flags._.length > 0) {
|
||||
flags.url = flags._[0].toString()
|
||||
}
|
||||
|
||||
const downloadQueueSize = parseInt(flags.queuesize)
|
||||
if (!downloadQueueSize || downloadQueueSize < 1) {
|
||||
console.log(`Unable to parse parse Queue Size: "${flags.queuesize}"`)
|
||||
Deno.exit()
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`deno
|
||||
--allow-net --allow-write --allow-read main.ts
|
||||
[--url] <url>
|
||||
[--output <downloadPath>]
|
||||
[--queuesize <int>]
|
||||
[--help]
|
||||
|
||||
parameters:
|
||||
[--url] <url>
|
||||
alis: -u
|
||||
required: true
|
||||
url to download
|
||||
|
||||
--output <downloadPath>
|
||||
alias: -o
|
||||
default: "."
|
||||
output path
|
||||
|
||||
--queuesize <queueSize>
|
||||
alias: -qs
|
||||
default: 5
|
||||
number of files to download simultaneously
|
||||
|
||||
--help
|
||||
alias: -h
|
||||
print help message
|
||||
`)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const downloadQueue = new DownloadQueue(5)
|
||||
const ui = new UI(downloadQueue)
|
||||
|
||||
// load all song details
|
||||
const playlistUrl = new URL(flags.url)
|
||||
const { playlist, images } = await fetchPlaylist(
|
||||
ui,
|
||||
downloadQueue,
|
||||
playlistUrl
|
||||
)
|
||||
|
||||
const albumName = playlist[0].album
|
||||
|
||||
const imageDownloadOptions: DownloadOptions[] = images.map(
|
||||
(imageUrl, i) => {
|
||||
const imagename = decodeURIComponent(
|
||||
imageUrl.pathname.split('/').pop() ?? `image_${i}`
|
||||
)
|
||||
const file = getFileDetails(flags.output, albumName, imagename)
|
||||
return {
|
||||
url: imageUrl,
|
||||
name: `Image - ${file.file}`,
|
||||
file,
|
||||
}
|
||||
}
|
||||
)
|
||||
downloadQueue.downloadBulk(...imageDownloadOptions)
|
||||
|
||||
for (const song of playlist) {
|
||||
downloadSong(ui, downloadQueue, song, flags.output)
|
||||
}
|
||||
|
||||
await downloadQueue.complete()
|
||||
ui.printLogs()
|
||||
}
|
||||
|
||||
if (!flags.url) {
|
||||
console.log('Missing URL\n\n')
|
||||
printHelp()
|
||||
} else if (flags.help) {
|
||||
printHelp()
|
||||
} else {
|
||||
main().catch((e) => console.error(e))
|
||||
}
|
||||
120
src/ui.ts
Normal file
120
src/ui.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
import { DownloadQueue } from './download.ts'
|
||||
import { getHeight, getWidth } from 'https://deno.land/x/terminal_size/mod.ts'
|
||||
import { humanizeBytes, round } from './util.ts'
|
||||
import { stdout } from 'node:process'
|
||||
|
||||
interface LogItem {
|
||||
date: Date
|
||||
message: string
|
||||
}
|
||||
|
||||
export class UI {
|
||||
private downloads: DownloadQueue
|
||||
private log_history: LogItem[] = []
|
||||
|
||||
constructor(downloads: DownloadQueue) {
|
||||
this.downloads = downloads
|
||||
this.printDownloads()
|
||||
downloads.on('update', (_) => this.printDownloads())
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
log(...message: any[]) {
|
||||
const msg = message.map((m) => m.toString()).join(' ')
|
||||
const date = new Date()
|
||||
this.log_history.push({ date, message: msg })
|
||||
this.printDownloads()
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
logExit(...message: any[]) {
|
||||
this.log(...message)
|
||||
this.printLogs()
|
||||
Deno.exit()
|
||||
}
|
||||
|
||||
printLogs() {
|
||||
for (const logItem of this.log_history) {
|
||||
const dateStr = logItem.date.toISOString()
|
||||
console.log(`${dateStr} ${logItem.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
private printDownloads() {
|
||||
const terminalWidth = getWidth()
|
||||
const terminaHeight = getHeight()
|
||||
// console.log(terminalWidth, terminaHeight)
|
||||
// process.exit()
|
||||
const status = this.downloads.status()
|
||||
let messageLines: string[] = []
|
||||
|
||||
// active queue
|
||||
for (const item of status.active) {
|
||||
const downloaded = humanizeBytes(item.downloaded)
|
||||
const total = item.size ? humanizeBytes(item.size) : '???'
|
||||
const percent = item.size
|
||||
? round((item.downloaded / item.size) * 100, 0)
|
||||
: '??'
|
||||
|
||||
{
|
||||
let nameLine = `${item.name}`
|
||||
const nameLineEnd = ` ${downloaded} / ${total} (${percent}%)`
|
||||
|
||||
const nameLineSpaces =
|
||||
terminalWidth - (nameLine.length + nameLineEnd.length)
|
||||
if (nameLineSpaces >= 0) {
|
||||
const spaces = new Array(nameLineSpaces).fill(' ').join('')
|
||||
nameLine = nameLine + spaces + nameLineEnd
|
||||
} else {
|
||||
nameLine =
|
||||
nameLine.slice(0, nameLine.length + nameLineSpaces) +
|
||||
nameLineEnd
|
||||
}
|
||||
|
||||
messageLines.push(nameLine)
|
||||
}
|
||||
|
||||
messageLines.push(` from: ${item.url.pathname}`)
|
||||
messageLines.push(
|
||||
` to: ${item.file ? item.file.relpath : 'Memory Buffer'}`
|
||||
)
|
||||
}
|
||||
|
||||
// other details
|
||||
messageLines.push(`___`)
|
||||
messageLines.push(
|
||||
`Active/Waiting/Completed = ${status.active.length}/${status.waiting}/${status.completed}`
|
||||
)
|
||||
|
||||
// trim away excess lines
|
||||
const messageStartIndex = messageLines.length - terminaHeight
|
||||
if (messageStartIndex >= 0) {
|
||||
messageLines = messageLines.slice(messageStartIndex)
|
||||
} else {
|
||||
messageLines = [
|
||||
...new Array(-messageStartIndex).fill(''),
|
||||
...messageLines,
|
||||
]
|
||||
}
|
||||
|
||||
// trim excess line widths
|
||||
messageLines = messageLines.map((line) => {
|
||||
if (line.length >= terminalWidth) {
|
||||
return line.slice(0, terminalWidth)
|
||||
} else {
|
||||
const spaces = new Array(terminalWidth - line.length)
|
||||
.fill(' ')
|
||||
.join('')
|
||||
return line + spaces
|
||||
}
|
||||
// return line
|
||||
})
|
||||
|
||||
// clear screen, go to top left
|
||||
// stdout.write("\x1b[2J")
|
||||
stdout.write(`\x1b[0;0H`)
|
||||
|
||||
// print lines
|
||||
stdout.write(messageLines.join('\n'))
|
||||
}
|
||||
}
|
||||
195
src/util.ts
Normal file
195
src/util.ts
Normal file
@@ -0,0 +1,195 @@
|
||||
import { resolve, join } from 'node:path'
|
||||
|
||||
const REGEX_UNSAFEFORFILE = /[\\\/:*?'"<>|*!?]|^null$|^\./gi
|
||||
|
||||
/** example
|
||||
* {
|
||||
* path: 'foldername',
|
||||
* fullpath: '/path/to/foldername',
|
||||
* location: '/path/to',
|
||||
*
|
||||
* file: 'filename',
|
||||
* ext: 'ext',
|
||||
* fullfile: 'filename.ext
|
||||
* relpath: 'foldername/filename.ext',
|
||||
* fullfilepath: '/path/to/foldername/filename.ext',
|
||||
* }
|
||||
*/
|
||||
export interface DownloadFolderDetails {
|
||||
path: string
|
||||
fullpath: string
|
||||
location: string
|
||||
}
|
||||
export interface DownloadFileDetails extends DownloadFolderDetails {
|
||||
file: string
|
||||
ext: string
|
||||
fullfile: string
|
||||
relpath: string
|
||||
fullfilepath: string
|
||||
}
|
||||
|
||||
export interface PlaylistSongData {
|
||||
url: URL
|
||||
album: string
|
||||
name: string
|
||||
track: number
|
||||
cd: number
|
||||
}
|
||||
|
||||
export function songFoldername(
|
||||
song: Partial<PlaylistSongData> | string
|
||||
): string {
|
||||
if (typeof song != 'string') {
|
||||
song = song.album ?? ''
|
||||
}
|
||||
|
||||
const albumname = song.replace(REGEX_UNSAFEFORFILE, '')
|
||||
return albumname
|
||||
}
|
||||
|
||||
export function songFilename(
|
||||
song: Partial<PlaylistSongData>
|
||||
): [filename: string, extension: string] {
|
||||
if (!song.name) {
|
||||
return ['', '']
|
||||
}
|
||||
|
||||
const songname = song.name.replace(REGEX_UNSAFEFORFILE, '')
|
||||
const tracknum = song.track ?? -1
|
||||
const cdnum = song.cd ?? -1
|
||||
|
||||
const cd = cdnum >= 0 ? cdnum + '.' : ''
|
||||
const track = tracknum >= 0 ? tracknum + '.' : ''
|
||||
const separator = cdnum >= 0 || tracknum >= 0 ? ' ' : ''
|
||||
|
||||
return [`${cd}${track}${separator}${songname}`, 'flac']
|
||||
|
||||
/*
|
||||
for example
|
||||
|
||||
song = {
|
||||
name: 'song',
|
||||
track: 1,
|
||||
cd: 1,
|
||||
}
|
||||
then filename = '1.1. song'
|
||||
|
||||
song = {
|
||||
name: 'song',
|
||||
track: 1,
|
||||
cd: -1,
|
||||
}
|
||||
then filename = '1. song'
|
||||
|
||||
song = {
|
||||
name: 'song',
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
then filename = 'song'
|
||||
*/
|
||||
}
|
||||
|
||||
export function splitFilename(
|
||||
filename: string
|
||||
): [filename: string, extension: string] {
|
||||
const spl = filename.split('.')
|
||||
let ext = 'bin'
|
||||
if (spl.length > 1) {
|
||||
ext = spl.pop() ?? ext
|
||||
}
|
||||
return [spl.join('.'), ext]
|
||||
}
|
||||
|
||||
export function getSongFileDetails(
|
||||
location: string,
|
||||
song: Partial<PlaylistSongData>
|
||||
): DownloadFileDetails {
|
||||
const path = songFoldername(song)
|
||||
const [file, ext] = songFilename(song)
|
||||
const fullfile = `${file}.${ext}`
|
||||
|
||||
const relpath = join(path, fullfile)
|
||||
|
||||
const fullpath = resolve(location, path)
|
||||
const fullfilepath = resolve(fullpath, fullfile)
|
||||
|
||||
return {
|
||||
path,
|
||||
file,
|
||||
ext,
|
||||
fullfile,
|
||||
relpath,
|
||||
fullpath,
|
||||
fullfilepath,
|
||||
location,
|
||||
}
|
||||
}
|
||||
|
||||
export function getFileDetails(
|
||||
location: string,
|
||||
albumName: string,
|
||||
filename: string
|
||||
): DownloadFileDetails {
|
||||
const path = songFoldername(albumName)
|
||||
|
||||
const [file, ext] = splitFilename(filename)
|
||||
const fullfile = `${file}.${ext}`
|
||||
|
||||
const relpath = join(path, fullfile)
|
||||
|
||||
const fullpath = resolve(location, path)
|
||||
const fullfilepath = resolve(fullpath, fullfile)
|
||||
|
||||
return {
|
||||
path,
|
||||
file,
|
||||
ext,
|
||||
fullfile,
|
||||
relpath,
|
||||
fullpath,
|
||||
fullfilepath,
|
||||
location,
|
||||
}
|
||||
}
|
||||
|
||||
const BYTE_MAGNITUDE = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
|
||||
export function humanizeBytes(size: number, decimals: number = 2) {
|
||||
let magnitude = 0
|
||||
while (size > 1000) {
|
||||
magnitude++
|
||||
size /= 1000
|
||||
}
|
||||
|
||||
size = round(size, decimals)
|
||||
|
||||
let magnitudeString = ''
|
||||
if (magnitude < BYTE_MAGNITUDE.length) {
|
||||
magnitudeString = BYTE_MAGNITUDE[magnitude]
|
||||
} else {
|
||||
const excessMagnitude = magnitude - BYTE_MAGNITUDE.length
|
||||
magnitudeString = `${
|
||||
BYTE_MAGNITUDE[BYTE_MAGNITUDE.length - 1]
|
||||
}+${excessMagnitude}`
|
||||
}
|
||||
|
||||
return `${size}${magnitudeString}`
|
||||
}
|
||||
|
||||
export function round(number: number, decimals: number = 2) {
|
||||
decimals = Math.max(0, Math.round(decimals))
|
||||
const decimalsMagnitude = Math.pow(10, decimals)
|
||||
return Math.round(number * decimalsMagnitude) / decimalsMagnitude
|
||||
}
|
||||
|
||||
const CHARS =
|
||||
'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.split('')
|
||||
export function randomChar() {
|
||||
return CHARS[Math.round(Math.random() * (CHARS.length - 1))]
|
||||
}
|
||||
export function randomChars(length: number) {
|
||||
return new Array(length)
|
||||
.fill('')
|
||||
.map((_) => randomChar())
|
||||
.join('')
|
||||
}
|
||||
241
src/web.ts
Normal file
241
src/web.ts
Normal file
@@ -0,0 +1,241 @@
|
||||
import { exists } from 'jsr:@std/fs/exists'
|
||||
import * as cheerio from 'https://esm.sh/cheerio?target=esnext'
|
||||
import { DownloadQueue } from './download.ts'
|
||||
import { UI } from './ui.ts'
|
||||
import {
|
||||
DownloadFileDetails,
|
||||
getSongFileDetails,
|
||||
PlaylistSongData,
|
||||
} from './util.ts'
|
||||
|
||||
const REGEX_PLAYLISTURL =
|
||||
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/?$/i
|
||||
const REGEX_SONGURL =
|
||||
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/[^\/]+\/?$/i
|
||||
const REGEX_ALBUMTITLE = /^(.*?) MP3 - Download/i
|
||||
|
||||
export interface PlaylistWhole {
|
||||
playlist: PlaylistSongData[]
|
||||
images: URL[]
|
||||
}
|
||||
|
||||
export async function fetchPlaylist(
|
||||
ui: UI,
|
||||
downloadQueue: DownloadQueue,
|
||||
url: URL
|
||||
): Promise<PlaylistWhole> {
|
||||
if (!REGEX_PLAYLISTURL.test(url.toString())) {
|
||||
throw `unaccepted url ${url}`
|
||||
}
|
||||
|
||||
ui.log(`downloading: ${url}`)
|
||||
|
||||
// load the playlist page's dom
|
||||
const req = await downloadQueue.downloadPromise({
|
||||
name: 'Playlist HTML',
|
||||
url,
|
||||
})
|
||||
const text = req.contentString()
|
||||
const $ = cheerio.load(text)
|
||||
|
||||
// get the album name from the page title
|
||||
const title =
|
||||
$.extract({
|
||||
title: 'title',
|
||||
}).title ?? ''
|
||||
const titleMatch = REGEX_ALBUMTITLE.exec(title)
|
||||
if (titleMatch === null) {
|
||||
throw `unable to grab album name from ${title}`
|
||||
}
|
||||
const albumName = titleMatch[1]
|
||||
ui.log(` title: ${albumName}`)
|
||||
|
||||
// get the album images from the page
|
||||
const images =
|
||||
$.extract({
|
||||
images: [
|
||||
{
|
||||
selector: '.albumImage a',
|
||||
value: 'href',
|
||||
},
|
||||
],
|
||||
}).images ?? []
|
||||
const imageUrls = images.map((img) => new URL(img))
|
||||
ui.log(` images: ${images.length}`)
|
||||
|
||||
// parse all rows in playlist
|
||||
const columns = getPlaylistTableHeaders($)
|
||||
const rows = getPlaylistRows($, columns, albumName)
|
||||
ui.log(` songs: ${rows.length}\n`)
|
||||
|
||||
return { playlist: rows, images: imageUrls }
|
||||
}
|
||||
|
||||
interface PlaylistTableColumns {
|
||||
name: number
|
||||
track: number
|
||||
cd: number
|
||||
}
|
||||
|
||||
function getPlaylistTableHeaders($: cheerio.CheerioAPI): PlaylistTableColumns {
|
||||
// get table header row
|
||||
const header = $('#songlist tr#songlist_header')
|
||||
const headerCells = header.extract({
|
||||
cells: ['th'],
|
||||
})
|
||||
|
||||
// get the index for specific columns
|
||||
const indexes = headerCells.cells.reduce(
|
||||
(p, c, i) => {
|
||||
// check the string content of the current cell
|
||||
switch (c.toLocaleLowerCase()) {
|
||||
case 'cd':
|
||||
p.cd = i
|
||||
break
|
||||
case '#':
|
||||
p.track = i
|
||||
break
|
||||
case 'song name':
|
||||
p.name = i
|
||||
}
|
||||
return p
|
||||
},
|
||||
// default values
|
||||
{
|
||||
name: -1,
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
)
|
||||
|
||||
if (indexes.name == -1) {
|
||||
throw 'unable to find song title column'
|
||||
}
|
||||
|
||||
return indexes
|
||||
}
|
||||
|
||||
function getPlaylistRows(
|
||||
$: cheerio.CheerioAPI,
|
||||
columns: PlaylistTableColumns,
|
||||
albumName: string
|
||||
): PlaylistSongData[] {
|
||||
const rows = $('#songlist tr:not(#songlist_header):not(#songlist_footer)')
|
||||
const rowsData: PlaylistSongData[] = []
|
||||
|
||||
// loop through each song in table
|
||||
rows.each((_, rowEl) => {
|
||||
const row = cheerio.load(rowEl)
|
||||
|
||||
// prase values from row
|
||||
const urlStr =
|
||||
row.extract({
|
||||
url: {
|
||||
selector: 'td.playlistDownloadSong a',
|
||||
value: 'href',
|
||||
},
|
||||
}).url ?? ''
|
||||
|
||||
const rowData: PlaylistSongData = {
|
||||
url: newKHInsiderURL(urlStr),
|
||||
album: albumName,
|
||||
name: '',
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
|
||||
const rowCells = row.extract({
|
||||
cells: ['td'],
|
||||
}).cells
|
||||
|
||||
rowData.name = rowCells[columns.name]
|
||||
if (!rowData.name) {
|
||||
throw `unable to grab song name from ${albumName} in row ${columns.name} - ${rowCells}`
|
||||
}
|
||||
|
||||
if (columns.track >= 0) {
|
||||
rowData.track = parseInt(rowCells[columns.track] ?? '-1')
|
||||
if (isNaN(rowData.track)) {
|
||||
rowData.track = -1
|
||||
}
|
||||
}
|
||||
|
||||
if (columns.cd >= 0) {
|
||||
rowData.cd = parseInt(rowCells[columns.cd] ?? '-1')
|
||||
if (isNaN(rowData.cd)) {
|
||||
rowData.cd = -1
|
||||
}
|
||||
}
|
||||
|
||||
rowsData.push(rowData)
|
||||
})
|
||||
|
||||
return rowsData
|
||||
}
|
||||
|
||||
function newKHInsiderURL(url: string) {
|
||||
if (!/^http/i.test(url.toString())) {
|
||||
return new URL('https://downloads.khinsider.com' + url)
|
||||
}
|
||||
return new URL(url)
|
||||
}
|
||||
|
||||
export async function downloadSong(
|
||||
ui: UI,
|
||||
downloadQueue: DownloadQueue,
|
||||
song: PlaylistSongData,
|
||||
location: string
|
||||
) {
|
||||
// get full url
|
||||
if (!REGEX_SONGURL.test(song.url.toString())) {
|
||||
throw `unaccepted url ${song.url}`
|
||||
}
|
||||
|
||||
const file = getSongFileDetails(location, song)
|
||||
|
||||
// skip file if it exists
|
||||
if (await exists(file.fullfilepath)) {
|
||||
ui.log(`skipping file already exists ${file.fullfilepath}`)
|
||||
return
|
||||
}
|
||||
|
||||
// get file url
|
||||
const flacURL = await getFileURL(downloadQueue, song, file)
|
||||
|
||||
ui.log(`downloading ${file.relpath}`)
|
||||
ui.log(` from ${flacURL}`)
|
||||
|
||||
// download the file
|
||||
downloadQueue.download({
|
||||
url: flacURL,
|
||||
name: `File - ${file.file}`,
|
||||
file,
|
||||
})
|
||||
}
|
||||
|
||||
async function getFileURL(
|
||||
downloadQueue: DownloadQueue,
|
||||
song: PlaylistSongData,
|
||||
file: DownloadFileDetails
|
||||
) {
|
||||
// load download page
|
||||
const resp = await downloadQueue.downloadPromise({
|
||||
url: song.url,
|
||||
name: `DL Page - ${file.file}`,
|
||||
})
|
||||
const text = resp.contentString()
|
||||
const $ = cheerio.load(text)
|
||||
|
||||
// extract the flac download link
|
||||
const flacUrlStr = $.extract({
|
||||
url: {
|
||||
selector: '#pageContent a[href*="flac"]',
|
||||
value: 'href',
|
||||
},
|
||||
}).url
|
||||
if (!flacUrlStr) {
|
||||
throw `can't find download link for ${song.url}`
|
||||
}
|
||||
|
||||
return new URL(flacUrlStr)
|
||||
}
|
||||
Reference in New Issue
Block a user