AslanG

Чтобы скачать файлы с сайта Akumu, вы можете использовать этот код на Node.js. Понадобятся два пакета: axios и cheerio. это медленно, но делает свою работу P.S. пропускает файл, если он присутствует и не изменен const axios = require("axios"); const cheerio = require("cheerio"); const fs = require("fs/promises"); const path = require("path"); const BASE_URL = "http://akumu.ru/lineage2/L2NA/P746/"; const OUTPUT_DIR = "./downloaded"; async function delay(ms) { return new Promise((res) => setTimeout(res, ms)); } async function downloadFile(url, filepath) { // Skip download if file already exists and was not modified today try { const stat = await fs.stat(filepath); const mtime = stat.mtime; const today = new Date(); if ( mtime.getFullYear() === today.getFullYear() && mtime.getMonth() === today.getMonth() && mtime.getDate() === today.getDate() ) { // File was modified today, do not skip } else { console.log(`File exists, skipping: ${filepath}`); return false; // Indicate skipped } } catch (e) { // File does not exist, proceed to download } const writer = await fs.open(filepath, "w"); const response = await axios({ url, method: "GET", responseType: "stream" }); const totalLength = parseInt(response.headers['content-length'], 10); let downloadedLength = 0; let lastLogged = Date.now(); let lastDownloaded = 0; const lastModified = response.headers["last-modified"]; const mtime = lastModified ? new Date(lastModified) : new Date(); return new Promise((resolve, reject) => { const stream = writer.createWriteStream(); response.data.on('data', (chunk) => { downloadedLength += chunk.length; const now = Date.now(); if (now - lastLogged > 1000 || downloadedLength === totalLength) { const percent = totalLength ? ((downloadedLength / totalLength) * 100).toFixed(2) : 'N/A'; const speed = ((downloadedLength - lastDownloaded) / ((now - lastLogged) / 1000)) / 1024; // KB/s process.stdout.write(`\rDownloading ${path.basename(filepath)}: ${percent}% (${(downloadedLength/1024).toFixed(1)} KB/${totalLength ? (totalLength/1024).toFixed(1) : '?'} KB) ${(speed).toFixed(1)} KB/s `); lastLogged = now; lastDownloaded = downloadedLength; } }); response.data.pipe(stream); response.data.on("end", async () => { await writer.close(); process.stdout.write("\n"); // Set mtime and atime try { await fs.utimes(filepath, mtime, mtime); } catch (e) { console.warn( `Failed to set file timestamp for ${filepath}:`, e.message ); } resolve(true); // Indicate downloaded }); response.data.on("error", async (err) => { await writer.close(); reject(err); }); }); } async function crawlAndDownload(url, localPath) { console.log("Visiting:", url); // Ensure local path exists await fs.mkdir(localPath, { recursive: true }); const { data } = await axios.get(url); const $ = cheerio.load(data); const links = $("tbody a") .toArray() .map((el) => $(el).attr("href")) .filter((href) => href && href !== "../"); for (const href of links) { const fullUrl = new URL(href, url).href; const decodedHref = decodeURIComponent(href); const targetPath = path.join(localPath, decodedHref); if (href.endsWith("/")) { // It's a folder, recurse normally await crawlAndDownload(fullUrl, targetPath); } else { // It's a file, retry download indefinitely on failure while (true) { try { console.log("Downloading file:", fullUrl); const downloaded = await downloadFile(fullUrl, targetPath); if (downloaded) { await delay(1000); // Delay only if file was downloaded } break; // Success or skipped, exit retry loop } catch (err) { console.error( `Failed to download ${fullUrl}, retrying... Error: ${err.message}` ); // Optionally wait before retrying to be gentle on server await delay(3000); } } } } } crawlAndDownload(BASE_URL, OUTPUT_DIR) .then(() => console.log("Done!")) .catch((err) => console.error("Error:", err));