php从文件读取url列表并下载 · php开发笔记

``` ~~~ <?php // download_with_structure.php // 设置不限制执行时间和内存 set_time_limit(0); ini_set('memory_limit', '712M'); // 配置参数 $inputFile = '2025550000.txt'; // URL列表文件 $outputBaseDir = 'ossfiles/'; // 下载基础目录 $batchSize = 5; // 每批处理数量 //$logFile = 'download_huisheng_log.txt'; // 日志文件 $logFile = 'download_huisheng_log2.txt'; // 日志文件 // 创建基础目录 if (!file_exists($outputBaseDir)) { mkdir($outputBaseDir, 0777, true); } // 获取当前批次参数 $start = isset($_GET['start']) ? (int)$_GET['start'] : 0; $urlList = file($inputFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); $totalUrls = count($urlList); // 处理当前批次 $batchUrls = array_slice($urlList, $start, $batchSize); $processed = 0; foreach ($batchUrls as $url) { $url = trim($url); $url = str_replace('?x-oss-process=image/resize','',$url); if (empty($url)) continue; try { $extension = getUrlExtension($url); // 返回 'jpg' if(in_array($extension,['flac','mp3','mp4','apk'] )){ //if(!in_array($extension,['flac','mp3','mp4'] )){ continue; } if(!stripos($url,'.oss')){ continue; } // 解析URL获取路径结构 $parsed = parse_url($url); $path = $parsed['path'] ?? ''; // 移除开头的斜杠并分割路径 $path = ltrim($path, '/'); $path = urldecode($path); $pathParts = explode('/', $path); $filename = array_pop($pathParts); $filename = urldecode($filename); // 构建本地目录结构 $localDir = $outputBaseDir; foreach ($pathParts as $dir) { $localDir .= $dir . '/'; if (!file_exists($localDir)) { mkdir($localDir, 0777, true); } } // 生成保存路径 $savePath = $localDir . $filename; // 如果文件已存在且大小>0则跳过 if (file_exists($savePath) && filesize($savePath) > 0) { logResult("文件已存在，跳过: {$savePath}"); $processed++; continue; } // 使用cURL下载文件 $ch = curl_init($url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); curl_setopt($ch, CURLOPT_TIMEOUT, 300); curl_setopt($ch, CURLOPT_NOPROGRESS, false); curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function($resource, $download_size, $downloaded, $upload_size, $uploaded) { // 每下载1MB重置时间限制 if ($downloaded && $downloaded % (1024*1024) === 0) { set_time_limit(30); } return 0; }); $fileContent = curl_exec($ch); $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); $error = curl_error($ch); curl_close($ch); if ($httpCode == 200 && $fileContent !== false) { if (file_put_contents($savePath, $fileContent)) { logResult("成功下载: {$url} -> {$savePath} (".number_format(strlen($fileContent))." bytes)"); $processed++; } else { throw new Exception("无法保存文件到 {$savePath}"); } } else { throw new Exception("下载失败，HTTP状态码: {$httpCode}, 错误: {$error}"); } } catch (Exception $e) { logResult("下载失败: {$url} - " . $e->getMessage()); } // 每条处理完后短暂延迟 usleep(200000); // 200毫秒延迟 } // 计算下一批次起始位置 $nextStart = $start + $batchSize; $remaining = max(0, $totalUrls - $nextStart); // 显示进度信息 echo "<!DOCTYPE html> <html> <head> <title>文件下载进度</title> <meta charset='UTF-8'> <style> body { font-family: Arial, sans-serif; margin: 20px; } .progress-container { width: 100%; background-color: #f1f1f1; } .progress-bar { width: ".min(100, round(($nextStart/$totalUrls)*100))."%; height: 30px; background-color: #4CAF50; text-align: center; line-height: 30px; color: white; } .stats { margin: 20px 0; padding: 10px; background: #f8f8f8; } </style> </head> <body> <h1>文件下载进度</h1> <div class='progress-container'> <div class='progress-bar'>".round(($nextStart/$totalUrls)*100, 1)."%</div> </div> <div class='stats'> <p>总文件数: {$totalUrls}</p> <p>已完成: {$nextStart}</p> <p>剩余: {$remaining}</p> <p>本批次成功下载: {$processed}/{$batchSize}</p> </div>"; if ($nextStart < $totalUrls) { $nextUrl = "extract_urls_down2.php?start={$nextStart}"; echo "<p><a href='{$nextUrl}' style='display: inline-block; padding: 10px 15px; background: #4CAF50; color: white; text-decoration: none;'>继续下载下一批 (".$batchSize."个)</a></p>"; // 自动跳转(1秒后) echo "<script>setTimeout(function(){ window.location.href = '{$nextUrl}'; }, 1000);</script>"; } else { echo "<p style='color: green; font-weight: bold;'>所有文件下载完成!</p>"; echo "<p><a href='{$outputBaseDir}' target='_blank'>浏览下载的文件</a></p>"; } echo "</body></html>"; // 日志记录函数 function logResult($message) { global $logFile; $timestamp = date('Y-m-d H:i:s'); $logEntry = "[{$timestamp}] {$message}" . PHP_EOL; file_put_contents($logFile, $logEntry, FILE_APPEND); echo ""; // 同时在HTML注释中输出便于调试 } function getUrlExtension($url) { $path = parse_url($url, PHP_URL_PATH); return pathinfo($path, PATHINFO_EXTENSION); } ?> ~~~ ```