created a download queue, added full screen status viewing
This commit is contained in:
302
src/download.ts
Normal file
302
src/download.ts
Normal file
@@ -0,0 +1,302 @@
|
||||
import EventEmitter from 'node:events'
|
||||
import { DownloadFileDetails, randomChars } from './util.ts'
|
||||
|
||||
export interface DownloadOptions {
|
||||
name: string
|
||||
url: URL
|
||||
file?: DownloadFileDetails
|
||||
}
|
||||
|
||||
export interface DownloadOptionsID extends DownloadOptions {
|
||||
id: string
|
||||
}
|
||||
|
||||
export interface DownloadItemStatus extends DownloadOptions {
|
||||
downloaded: number
|
||||
size?: number
|
||||
}
|
||||
|
||||
export interface DownloadStatus {
|
||||
waiting: number
|
||||
completed: number
|
||||
active: DownloadItemStatus[]
|
||||
}
|
||||
|
||||
export interface DownloadQueueEvents {
|
||||
update: (item: DownloadQueue) => void
|
||||
done: (item: DownloadItem) => void
|
||||
complete: () => void
|
||||
error: (message: string) => void
|
||||
}
|
||||
|
||||
export declare interface DownloadQueue {
|
||||
on<U extends keyof DownloadQueueEvents>(
|
||||
event: U,
|
||||
listener: DownloadQueueEvents[U]
|
||||
): this
|
||||
on(event: `done_${string}`, listener: DownloadQueueEvents['done']): this
|
||||
|
||||
once<U extends keyof DownloadQueueEvents>(
|
||||
event: U,
|
||||
listener: DownloadQueueEvents[U]
|
||||
): this
|
||||
once(event: `done_${string}`, listener: DownloadQueueEvents['done']): this
|
||||
|
||||
emit<U extends keyof DownloadQueueEvents>(
|
||||
event: U,
|
||||
...args: Parameters<DownloadQueueEvents[U]>
|
||||
): boolean
|
||||
emit(
|
||||
event: `done_${string}`,
|
||||
...args: Parameters<DownloadQueueEvents['done']>
|
||||
): this
|
||||
}
|
||||
|
||||
export class DownloadQueue extends EventEmitter {
|
||||
private queue_waiting: DownloadOptionsID[] = []
|
||||
private queue_active: DownloadItem[] = []
|
||||
private queue_history: DownloadItem[] = []
|
||||
private queue_size = 1
|
||||
private checking_queue = false
|
||||
|
||||
constructor(queue_size: number) {
|
||||
super()
|
||||
if (queue_size > 1) {
|
||||
this.queue_size = queue_size
|
||||
}
|
||||
}
|
||||
|
||||
downloadBulk(...options: DownloadOptions[]): string[] {
|
||||
const ids: string[] = []
|
||||
this.queue_waiting.push(
|
||||
...options.map((opt) => {
|
||||
const id = this.generateID()
|
||||
ids.push(id)
|
||||
return { ...opt, id }
|
||||
})
|
||||
)
|
||||
this.checkQueue()
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
download(options: DownloadOptions): string {
|
||||
const id = this.generateID()
|
||||
this.queue_waiting.push({ ...options, id })
|
||||
this.checkQueue()
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
downloadPromise(options: DownloadOptions): Promise<DownloadItem> {
|
||||
const id = this.download(options)
|
||||
return new Promise((resolve) => {
|
||||
this.once(`done_${id}`, (item) => {
|
||||
resolve(item)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
complete(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
this.on('complete', () => resolve())
|
||||
})
|
||||
}
|
||||
|
||||
private IDExists(id: string): boolean {
|
||||
const waitingIds = this.queue_waiting.map((item) => item.id)
|
||||
const queuedIds = [...this.queue_active, ...this.queue_history].map(
|
||||
(item) => item.options.id
|
||||
)
|
||||
return [...waitingIds, ...queuedIds].some((itemId) => itemId === id)
|
||||
}
|
||||
|
||||
private generateID(): string {
|
||||
let id = ''
|
||||
do {
|
||||
id = randomChars(16)
|
||||
} while (this.IDExists(id))
|
||||
return id
|
||||
}
|
||||
|
||||
status(): DownloadStatus {
|
||||
return {
|
||||
waiting: this.queue_waiting.length,
|
||||
completed: this.queue_history.length,
|
||||
active: this.queue_active.map((item) => ({
|
||||
...item.options,
|
||||
downloaded: item.downloaded,
|
||||
size: item.contentlength,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
private checkQueue() {
|
||||
if (this.checking_queue) {
|
||||
return
|
||||
}
|
||||
|
||||
this.checking_queue = true
|
||||
|
||||
// update active queue
|
||||
const queue_active: DownloadItem[] = []
|
||||
for (const item of this.queue_active) {
|
||||
if (item.completed) {
|
||||
this.queue_history.push(item)
|
||||
} else if (item.errored) {
|
||||
this.queue_waiting.push(item.options)
|
||||
} else {
|
||||
queue_active.push(item)
|
||||
}
|
||||
}
|
||||
this.queue_active = queue_active
|
||||
|
||||
// add to active queue
|
||||
while (
|
||||
this.queue_active.length < this.queue_size &&
|
||||
this.queue_waiting.length > 0
|
||||
) {
|
||||
const item = this.queue_waiting.shift()
|
||||
if (!item) {
|
||||
break
|
||||
}
|
||||
|
||||
this.queue_active.push(this.startDownloadItem(item))
|
||||
}
|
||||
|
||||
this.checking_queue = false
|
||||
this.emit('update', this)
|
||||
|
||||
if (this.queue_active.length === 0 && this.queue_waiting.length === 0) {
|
||||
this.emit('complete')
|
||||
}
|
||||
}
|
||||
|
||||
private startDownloadItem(item: DownloadOptionsID): DownloadItem {
|
||||
const download = new DownloadItem(item)
|
||||
download.on('error', (err) => {
|
||||
this.emit('error', err)
|
||||
this.checkQueue()
|
||||
})
|
||||
download.on('chunk', (_) => {
|
||||
this.emit('update', this)
|
||||
})
|
||||
download.on('done', (d) => {
|
||||
this.emit('done', d)
|
||||
this.emit(`done_${d.options.id}`, d)
|
||||
this.checkQueue()
|
||||
})
|
||||
return download
|
||||
}
|
||||
}
|
||||
|
||||
export interface DownloadItemEvents {
|
||||
chunk: (item: DownloadItem) => void
|
||||
done: (item: DownloadItem) => void
|
||||
error: (message: string) => void
|
||||
}
|
||||
|
||||
export declare interface DownloadItem {
|
||||
on<U extends keyof DownloadItemEvents>(
|
||||
event: U,
|
||||
listener: DownloadItemEvents[U]
|
||||
): this
|
||||
once<U extends keyof DownloadItemEvents>(
|
||||
event: U,
|
||||
listener: DownloadItemEvents[U]
|
||||
): this
|
||||
emit<U extends keyof DownloadItemEvents>(
|
||||
event: U,
|
||||
...args: Parameters<DownloadItemEvents[U]>
|
||||
): boolean
|
||||
}
|
||||
|
||||
export class DownloadItem extends EventEmitter {
|
||||
options: DownloadOptionsID
|
||||
downloaded: number = 0
|
||||
contentlength?: number
|
||||
content = new Uint8Array(0)
|
||||
|
||||
completed = false
|
||||
errored = false
|
||||
|
||||
constructor(options: DownloadOptionsID) {
|
||||
super()
|
||||
this.options = options
|
||||
this.startDownload()
|
||||
.then(() => {
|
||||
this.completed = true
|
||||
this.emit('done', this)
|
||||
})
|
||||
.catch((err) => {
|
||||
this.errored = true
|
||||
this.emit('error', err)
|
||||
})
|
||||
}
|
||||
|
||||
contentString(): string {
|
||||
if (this.options.file || !this.completed) {
|
||||
return ''
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder('utf-8')
|
||||
return decoder.decode(this.content)
|
||||
}
|
||||
|
||||
private updateContentLength(res: Response) {
|
||||
const length = parseInt(res.headers.get('Content-Length') ?? '')
|
||||
if (length && length > 0) {
|
||||
this.contentlength = length
|
||||
}
|
||||
}
|
||||
|
||||
private async startDownload() {
|
||||
try {
|
||||
const res = await fetch(this.options.url)
|
||||
if (res.status !== 200) {
|
||||
throw `HTTP status code: ${res.status} ${res.statusText}`
|
||||
}
|
||||
|
||||
if (!res.body) {
|
||||
throw `HTTP response has no body`
|
||||
}
|
||||
|
||||
this.updateContentLength(res)
|
||||
this.emit('chunk', this)
|
||||
|
||||
if (this.options.file) {
|
||||
await Deno.mkdir(this.options.file.fullpath, {
|
||||
recursive: true,
|
||||
})
|
||||
const file = await Deno.open(this.options.file.fullfilepath, {
|
||||
create: true,
|
||||
write: true,
|
||||
})
|
||||
|
||||
for await (const chunk of res.body) {
|
||||
await file.write(chunk)
|
||||
this.downloaded += chunk.length
|
||||
|
||||
this.emit('chunk', this)
|
||||
}
|
||||
} else {
|
||||
const chunks = []
|
||||
for await (const chunk of res.body) {
|
||||
chunks.push(chunk)
|
||||
this.downloaded += chunk.length
|
||||
|
||||
this.emit('chunk', this)
|
||||
}
|
||||
|
||||
this.content = new Uint8Array(this.downloaded)
|
||||
let offset = 0
|
||||
for (const chunk of chunks) {
|
||||
this.content.set(chunk, offset)
|
||||
offset += chunk.length
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
108
src/main.ts
Normal file
108
src/main.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
import { parseArgs } from 'jsr:@std/cli/parse-args'
|
||||
import { downloadSong, fetchPlaylist } from './web.ts'
|
||||
import { DownloadOptions, DownloadQueue } from './download.ts'
|
||||
import { UI } from './ui.ts'
|
||||
import { getFileDetails } from './util.ts'
|
||||
|
||||
// parse args
|
||||
const flags = parseArgs(Deno.args, {
|
||||
alias: {
|
||||
url: ['u'],
|
||||
output: ['o'],
|
||||
queuesize: ['qs'],
|
||||
help: ['h'],
|
||||
},
|
||||
string: ['url', 'output', 'queuesize'],
|
||||
boolean: ['help'],
|
||||
default: {
|
||||
url: '',
|
||||
output: '.',
|
||||
queuesize: '5',
|
||||
},
|
||||
})
|
||||
|
||||
if (!flags.url && flags._.length > 0) {
|
||||
flags.url = flags._[0].toString()
|
||||
}
|
||||
|
||||
const downloadQueueSize = parseInt(flags.queuesize)
|
||||
if (!downloadQueueSize || downloadQueueSize < 1) {
|
||||
console.log(`Unable to parse parse Queue Size: "${flags.queuesize}"`)
|
||||
Deno.exit()
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`deno
|
||||
--allow-net --allow-write --allow-read main.ts
|
||||
[--url] <url>
|
||||
[--output <downloadPath>]
|
||||
[--queuesize <int>]
|
||||
[--help]
|
||||
|
||||
parameters:
|
||||
[--url] <url>
|
||||
alis: -u
|
||||
required: true
|
||||
url to download
|
||||
|
||||
--output <downloadPath>
|
||||
alias: -o
|
||||
default: "."
|
||||
output path
|
||||
|
||||
--queuesize <queueSize>
|
||||
alias: -qs
|
||||
default: 5
|
||||
number of files to download simultaneously
|
||||
|
||||
--help
|
||||
alias: -h
|
||||
print help message
|
||||
`)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const downloadQueue = new DownloadQueue(5)
|
||||
const ui = new UI(downloadQueue)
|
||||
|
||||
// load all song details
|
||||
const playlistUrl = new URL(flags.url)
|
||||
const { playlist, images } = await fetchPlaylist(
|
||||
ui,
|
||||
downloadQueue,
|
||||
playlistUrl
|
||||
)
|
||||
|
||||
const albumName = playlist[0].album
|
||||
|
||||
const imageDownloadOptions: DownloadOptions[] = images.map(
|
||||
(imageUrl, i) => {
|
||||
const imagename = decodeURIComponent(
|
||||
imageUrl.pathname.split('/').pop() ?? `image_${i}`
|
||||
)
|
||||
const file = getFileDetails(flags.output, albumName, imagename)
|
||||
return {
|
||||
url: imageUrl,
|
||||
name: `Image - ${file.file}`,
|
||||
file,
|
||||
}
|
||||
}
|
||||
)
|
||||
downloadQueue.downloadBulk(...imageDownloadOptions)
|
||||
|
||||
for (const song of playlist) {
|
||||
downloadSong(ui, downloadQueue, song, flags.output)
|
||||
}
|
||||
|
||||
await downloadQueue.complete()
|
||||
ui.printLogs()
|
||||
}
|
||||
|
||||
if (!flags.url) {
|
||||
console.log('Missing URL\n\n')
|
||||
printHelp()
|
||||
} else if (flags.help) {
|
||||
printHelp()
|
||||
} else {
|
||||
main().catch((e) => console.error(e))
|
||||
}
|
||||
120
src/ui.ts
Normal file
120
src/ui.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
import { DownloadQueue } from './download.ts'
|
||||
import { getHeight, getWidth } from 'https://deno.land/x/terminal_size/mod.ts'
|
||||
import { humanizeBytes, round } from './util.ts'
|
||||
import { stdout } from 'node:process'
|
||||
|
||||
interface LogItem {
|
||||
date: Date
|
||||
message: string
|
||||
}
|
||||
|
||||
export class UI {
|
||||
private downloads: DownloadQueue
|
||||
private log_history: LogItem[] = []
|
||||
|
||||
constructor(downloads: DownloadQueue) {
|
||||
this.downloads = downloads
|
||||
this.printDownloads()
|
||||
downloads.on('update', (_) => this.printDownloads())
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
log(...message: any[]) {
|
||||
const msg = message.map((m) => m.toString()).join(' ')
|
||||
const date = new Date()
|
||||
this.log_history.push({ date, message: msg })
|
||||
this.printDownloads()
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
logExit(...message: any[]) {
|
||||
this.log(...message)
|
||||
this.printLogs()
|
||||
Deno.exit()
|
||||
}
|
||||
|
||||
printLogs() {
|
||||
for (const logItem of this.log_history) {
|
||||
const dateStr = logItem.date.toISOString()
|
||||
console.log(`${dateStr} ${logItem.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
private printDownloads() {
|
||||
const terminalWidth = getWidth()
|
||||
const terminaHeight = getHeight()
|
||||
// console.log(terminalWidth, terminaHeight)
|
||||
// process.exit()
|
||||
const status = this.downloads.status()
|
||||
let messageLines: string[] = []
|
||||
|
||||
// active queue
|
||||
for (const item of status.active) {
|
||||
const downloaded = humanizeBytes(item.downloaded)
|
||||
const total = item.size ? humanizeBytes(item.size) : '???'
|
||||
const percent = item.size
|
||||
? round((item.downloaded / item.size) * 100, 0)
|
||||
: '??'
|
||||
|
||||
{
|
||||
let nameLine = `${item.name}`
|
||||
const nameLineEnd = ` ${downloaded} / ${total} (${percent}%)`
|
||||
|
||||
const nameLineSpaces =
|
||||
terminalWidth - (nameLine.length + nameLineEnd.length)
|
||||
if (nameLineSpaces >= 0) {
|
||||
const spaces = new Array(nameLineSpaces).fill(' ').join('')
|
||||
nameLine = nameLine + spaces + nameLineEnd
|
||||
} else {
|
||||
nameLine =
|
||||
nameLine.slice(0, nameLine.length + nameLineSpaces) +
|
||||
nameLineEnd
|
||||
}
|
||||
|
||||
messageLines.push(nameLine)
|
||||
}
|
||||
|
||||
messageLines.push(` from: ${item.url.pathname}`)
|
||||
messageLines.push(
|
||||
` to: ${item.file ? item.file.relpath : 'Memory Buffer'}`
|
||||
)
|
||||
}
|
||||
|
||||
// other details
|
||||
messageLines.push(`___`)
|
||||
messageLines.push(
|
||||
`Active/Waiting/Completed = ${status.active.length}/${status.waiting}/${status.completed}`
|
||||
)
|
||||
|
||||
// trim away excess lines
|
||||
const messageStartIndex = messageLines.length - terminaHeight
|
||||
if (messageStartIndex >= 0) {
|
||||
messageLines = messageLines.slice(messageStartIndex)
|
||||
} else {
|
||||
messageLines = [
|
||||
...new Array(-messageStartIndex).fill(''),
|
||||
...messageLines,
|
||||
]
|
||||
}
|
||||
|
||||
// trim excess line widths
|
||||
messageLines = messageLines.map((line) => {
|
||||
if (line.length >= terminalWidth) {
|
||||
return line.slice(0, terminalWidth)
|
||||
} else {
|
||||
const spaces = new Array(terminalWidth - line.length)
|
||||
.fill(' ')
|
||||
.join('')
|
||||
return line + spaces
|
||||
}
|
||||
// return line
|
||||
})
|
||||
|
||||
// clear screen, go to top left
|
||||
// stdout.write("\x1b[2J")
|
||||
stdout.write(`\x1b[0;0H`)
|
||||
|
||||
// print lines
|
||||
stdout.write(messageLines.join('\n'))
|
||||
}
|
||||
}
|
||||
195
src/util.ts
Normal file
195
src/util.ts
Normal file
@@ -0,0 +1,195 @@
|
||||
import { resolve, join } from 'node:path'
|
||||
|
||||
const REGEX_UNSAFEFORFILE = /[\\\/:*?'"<>|*!?]|^null$|^\./gi
|
||||
|
||||
/** example
|
||||
* {
|
||||
* path: 'foldername',
|
||||
* fullpath: '/path/to/foldername',
|
||||
* location: '/path/to',
|
||||
*
|
||||
* file: 'filename',
|
||||
* ext: 'ext',
|
||||
* fullfile: 'filename.ext
|
||||
* relpath: 'foldername/filename.ext',
|
||||
* fullfilepath: '/path/to/foldername/filename.ext',
|
||||
* }
|
||||
*/
|
||||
export interface DownloadFolderDetails {
|
||||
path: string
|
||||
fullpath: string
|
||||
location: string
|
||||
}
|
||||
export interface DownloadFileDetails extends DownloadFolderDetails {
|
||||
file: string
|
||||
ext: string
|
||||
fullfile: string
|
||||
relpath: string
|
||||
fullfilepath: string
|
||||
}
|
||||
|
||||
export interface PlaylistSongData {
|
||||
url: URL
|
||||
album: string
|
||||
name: string
|
||||
track: number
|
||||
cd: number
|
||||
}
|
||||
|
||||
export function songFoldername(
|
||||
song: Partial<PlaylistSongData> | string
|
||||
): string {
|
||||
if (typeof song != 'string') {
|
||||
song = song.album ?? ''
|
||||
}
|
||||
|
||||
const albumname = song.replace(REGEX_UNSAFEFORFILE, '')
|
||||
return albumname
|
||||
}
|
||||
|
||||
export function songFilename(
|
||||
song: Partial<PlaylistSongData>
|
||||
): [filename: string, extension: string] {
|
||||
if (!song.name) {
|
||||
return ['', '']
|
||||
}
|
||||
|
||||
const songname = song.name.replace(REGEX_UNSAFEFORFILE, '')
|
||||
const tracknum = song.track ?? -1
|
||||
const cdnum = song.cd ?? -1
|
||||
|
||||
const cd = cdnum >= 0 ? cdnum + '.' : ''
|
||||
const track = tracknum >= 0 ? tracknum + '.' : ''
|
||||
const separator = cdnum >= 0 || tracknum >= 0 ? ' ' : ''
|
||||
|
||||
return [`${cd}${track}${separator}${songname}`, 'flac']
|
||||
|
||||
/*
|
||||
for example
|
||||
|
||||
song = {
|
||||
name: 'song',
|
||||
track: 1,
|
||||
cd: 1,
|
||||
}
|
||||
then filename = '1.1. song'
|
||||
|
||||
song = {
|
||||
name: 'song',
|
||||
track: 1,
|
||||
cd: -1,
|
||||
}
|
||||
then filename = '1. song'
|
||||
|
||||
song = {
|
||||
name: 'song',
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
then filename = 'song'
|
||||
*/
|
||||
}
|
||||
|
||||
export function splitFilename(
|
||||
filename: string
|
||||
): [filename: string, extension: string] {
|
||||
const spl = filename.split('.')
|
||||
let ext = 'bin'
|
||||
if (spl.length > 1) {
|
||||
ext = spl.pop() ?? ext
|
||||
}
|
||||
return [spl.join('.'), ext]
|
||||
}
|
||||
|
||||
export function getSongFileDetails(
|
||||
location: string,
|
||||
song: Partial<PlaylistSongData>
|
||||
): DownloadFileDetails {
|
||||
const path = songFoldername(song)
|
||||
const [file, ext] = songFilename(song)
|
||||
const fullfile = `${file}.${ext}`
|
||||
|
||||
const relpath = join(path, fullfile)
|
||||
|
||||
const fullpath = resolve(location, path)
|
||||
const fullfilepath = resolve(fullpath, fullfile)
|
||||
|
||||
return {
|
||||
path,
|
||||
file,
|
||||
ext,
|
||||
fullfile,
|
||||
relpath,
|
||||
fullpath,
|
||||
fullfilepath,
|
||||
location,
|
||||
}
|
||||
}
|
||||
|
||||
export function getFileDetails(
|
||||
location: string,
|
||||
albumName: string,
|
||||
filename: string
|
||||
): DownloadFileDetails {
|
||||
const path = songFoldername(albumName)
|
||||
|
||||
const [file, ext] = splitFilename(filename)
|
||||
const fullfile = `${file}.${ext}`
|
||||
|
||||
const relpath = join(path, fullfile)
|
||||
|
||||
const fullpath = resolve(location, path)
|
||||
const fullfilepath = resolve(fullpath, fullfile)
|
||||
|
||||
return {
|
||||
path,
|
||||
file,
|
||||
ext,
|
||||
fullfile,
|
||||
relpath,
|
||||
fullpath,
|
||||
fullfilepath,
|
||||
location,
|
||||
}
|
||||
}
|
||||
|
||||
const BYTE_MAGNITUDE = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
|
||||
export function humanizeBytes(size: number, decimals: number = 2) {
|
||||
let magnitude = 0
|
||||
while (size > 1000) {
|
||||
magnitude++
|
||||
size /= 1000
|
||||
}
|
||||
|
||||
size = round(size, decimals)
|
||||
|
||||
let magnitudeString = ''
|
||||
if (magnitude < BYTE_MAGNITUDE.length) {
|
||||
magnitudeString = BYTE_MAGNITUDE[magnitude]
|
||||
} else {
|
||||
const excessMagnitude = magnitude - BYTE_MAGNITUDE.length
|
||||
magnitudeString = `${
|
||||
BYTE_MAGNITUDE[BYTE_MAGNITUDE.length - 1]
|
||||
}+${excessMagnitude}`
|
||||
}
|
||||
|
||||
return `${size}${magnitudeString}`
|
||||
}
|
||||
|
||||
export function round(number: number, decimals: number = 2) {
|
||||
decimals = Math.max(0, Math.round(decimals))
|
||||
const decimalsMagnitude = Math.pow(10, decimals)
|
||||
return Math.round(number * decimalsMagnitude) / decimalsMagnitude
|
||||
}
|
||||
|
||||
const CHARS =
|
||||
'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.split('')
|
||||
export function randomChar() {
|
||||
return CHARS[Math.round(Math.random() * (CHARS.length - 1))]
|
||||
}
|
||||
export function randomChars(length: number) {
|
||||
return new Array(length)
|
||||
.fill('')
|
||||
.map((_) => randomChar())
|
||||
.join('')
|
||||
}
|
||||
241
src/web.ts
Normal file
241
src/web.ts
Normal file
@@ -0,0 +1,241 @@
|
||||
import { exists } from 'jsr:@std/fs/exists'
|
||||
import * as cheerio from 'https://esm.sh/cheerio?target=esnext'
|
||||
import { DownloadQueue } from './download.ts'
|
||||
import { UI } from './ui.ts'
|
||||
import {
|
||||
DownloadFileDetails,
|
||||
getSongFileDetails,
|
||||
PlaylistSongData,
|
||||
} from './util.ts'
|
||||
|
||||
const REGEX_PLAYLISTURL =
|
||||
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/?$/i
|
||||
const REGEX_SONGURL =
|
||||
/^https:\/\/downloads\.khinsider\.com\/game-soundtracks\/album\/[^\/]+\/[^\/]+\/?$/i
|
||||
const REGEX_ALBUMTITLE = /^(.*?) MP3 - Download/i
|
||||
|
||||
export interface PlaylistWhole {
|
||||
playlist: PlaylistSongData[]
|
||||
images: URL[]
|
||||
}
|
||||
|
||||
export async function fetchPlaylist(
|
||||
ui: UI,
|
||||
downloadQueue: DownloadQueue,
|
||||
url: URL
|
||||
): Promise<PlaylistWhole> {
|
||||
if (!REGEX_PLAYLISTURL.test(url.toString())) {
|
||||
throw `unaccepted url ${url}`
|
||||
}
|
||||
|
||||
ui.log(`downloading: ${url}`)
|
||||
|
||||
// load the playlist page's dom
|
||||
const req = await downloadQueue.downloadPromise({
|
||||
name: 'Playlist HTML',
|
||||
url,
|
||||
})
|
||||
const text = req.contentString()
|
||||
const $ = cheerio.load(text)
|
||||
|
||||
// get the album name from the page title
|
||||
const title =
|
||||
$.extract({
|
||||
title: 'title',
|
||||
}).title ?? ''
|
||||
const titleMatch = REGEX_ALBUMTITLE.exec(title)
|
||||
if (titleMatch === null) {
|
||||
throw `unable to grab album name from ${title}`
|
||||
}
|
||||
const albumName = titleMatch[1]
|
||||
ui.log(` title: ${albumName}`)
|
||||
|
||||
// get the album images from the page
|
||||
const images =
|
||||
$.extract({
|
||||
images: [
|
||||
{
|
||||
selector: '.albumImage a',
|
||||
value: 'href',
|
||||
},
|
||||
],
|
||||
}).images ?? []
|
||||
const imageUrls = images.map((img) => new URL(img))
|
||||
ui.log(` images: ${images.length}`)
|
||||
|
||||
// parse all rows in playlist
|
||||
const columns = getPlaylistTableHeaders($)
|
||||
const rows = getPlaylistRows($, columns, albumName)
|
||||
ui.log(` songs: ${rows.length}\n`)
|
||||
|
||||
return { playlist: rows, images: imageUrls }
|
||||
}
|
||||
|
||||
interface PlaylistTableColumns {
|
||||
name: number
|
||||
track: number
|
||||
cd: number
|
||||
}
|
||||
|
||||
function getPlaylistTableHeaders($: cheerio.CheerioAPI): PlaylistTableColumns {
|
||||
// get table header row
|
||||
const header = $('#songlist tr#songlist_header')
|
||||
const headerCells = header.extract({
|
||||
cells: ['th'],
|
||||
})
|
||||
|
||||
// get the index for specific columns
|
||||
const indexes = headerCells.cells.reduce(
|
||||
(p, c, i) => {
|
||||
// check the string content of the current cell
|
||||
switch (c.toLocaleLowerCase()) {
|
||||
case 'cd':
|
||||
p.cd = i
|
||||
break
|
||||
case '#':
|
||||
p.track = i
|
||||
break
|
||||
case 'song name':
|
||||
p.name = i
|
||||
}
|
||||
return p
|
||||
},
|
||||
// default values
|
||||
{
|
||||
name: -1,
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
)
|
||||
|
||||
if (indexes.name == -1) {
|
||||
throw 'unable to find song title column'
|
||||
}
|
||||
|
||||
return indexes
|
||||
}
|
||||
|
||||
function getPlaylistRows(
|
||||
$: cheerio.CheerioAPI,
|
||||
columns: PlaylistTableColumns,
|
||||
albumName: string
|
||||
): PlaylistSongData[] {
|
||||
const rows = $('#songlist tr:not(#songlist_header):not(#songlist_footer)')
|
||||
const rowsData: PlaylistSongData[] = []
|
||||
|
||||
// loop through each song in table
|
||||
rows.each((_, rowEl) => {
|
||||
const row = cheerio.load(rowEl)
|
||||
|
||||
// prase values from row
|
||||
const urlStr =
|
||||
row.extract({
|
||||
url: {
|
||||
selector: 'td.playlistDownloadSong a',
|
||||
value: 'href',
|
||||
},
|
||||
}).url ?? ''
|
||||
|
||||
const rowData: PlaylistSongData = {
|
||||
url: newKHInsiderURL(urlStr),
|
||||
album: albumName,
|
||||
name: '',
|
||||
track: -1,
|
||||
cd: -1,
|
||||
}
|
||||
|
||||
const rowCells = row.extract({
|
||||
cells: ['td'],
|
||||
}).cells
|
||||
|
||||
rowData.name = rowCells[columns.name]
|
||||
if (!rowData.name) {
|
||||
throw `unable to grab song name from ${albumName} in row ${columns.name} - ${rowCells}`
|
||||
}
|
||||
|
||||
if (columns.track >= 0) {
|
||||
rowData.track = parseInt(rowCells[columns.track] ?? '-1')
|
||||
if (isNaN(rowData.track)) {
|
||||
rowData.track = -1
|
||||
}
|
||||
}
|
||||
|
||||
if (columns.cd >= 0) {
|
||||
rowData.cd = parseInt(rowCells[columns.cd] ?? '-1')
|
||||
if (isNaN(rowData.cd)) {
|
||||
rowData.cd = -1
|
||||
}
|
||||
}
|
||||
|
||||
rowsData.push(rowData)
|
||||
})
|
||||
|
||||
return rowsData
|
||||
}
|
||||
|
||||
function newKHInsiderURL(url: string) {
|
||||
if (!/^http/i.test(url.toString())) {
|
||||
return new URL('https://downloads.khinsider.com' + url)
|
||||
}
|
||||
return new URL(url)
|
||||
}
|
||||
|
||||
export async function downloadSong(
|
||||
ui: UI,
|
||||
downloadQueue: DownloadQueue,
|
||||
song: PlaylistSongData,
|
||||
location: string
|
||||
) {
|
||||
// get full url
|
||||
if (!REGEX_SONGURL.test(song.url.toString())) {
|
||||
throw `unaccepted url ${song.url}`
|
||||
}
|
||||
|
||||
const file = getSongFileDetails(location, song)
|
||||
|
||||
// skip file if it exists
|
||||
if (await exists(file.fullfilepath)) {
|
||||
ui.log(`skipping file already exists ${file.fullfilepath}`)
|
||||
return
|
||||
}
|
||||
|
||||
// get file url
|
||||
const flacURL = await getFileURL(downloadQueue, song, file)
|
||||
|
||||
ui.log(`downloading ${file.relpath}`)
|
||||
ui.log(` from ${flacURL}`)
|
||||
|
||||
// download the file
|
||||
downloadQueue.download({
|
||||
url: flacURL,
|
||||
name: `File - ${file.file}`,
|
||||
file,
|
||||
})
|
||||
}
|
||||
|
||||
async function getFileURL(
|
||||
downloadQueue: DownloadQueue,
|
||||
song: PlaylistSongData,
|
||||
file: DownloadFileDetails
|
||||
) {
|
||||
// load download page
|
||||
const resp = await downloadQueue.downloadPromise({
|
||||
url: song.url,
|
||||
name: `DL Page - ${file.file}`,
|
||||
})
|
||||
const text = resp.contentString()
|
||||
const $ = cheerio.load(text)
|
||||
|
||||
// extract the flac download link
|
||||
const flacUrlStr = $.extract({
|
||||
url: {
|
||||
selector: '#pageContent a[href*="flac"]',
|
||||
value: 'href',
|
||||
},
|
||||
}).url
|
||||
if (!flacUrlStr) {
|
||||
throw `can't find download link for ${song.url}`
|
||||
}
|
||||
|
||||
return new URL(flacUrlStr)
|
||||
}
|
||||
Reference in New Issue
Block a user