1 use log::info; 2 use regex::Regex; 3 use reqwest::Url; 4 use serde::{Deserialize, Serialize}; 5 use std::os::unix::fs::PermissionsExt; 6 use std::{ 7 fs::File, 8 path::PathBuf, 9 process::{Command, Stdio}, 10 }; 11 use zip::ZipArchive; 12 13 use crate::utils::{file::FileUtils, stdio::StdioUtils}; 14 15 use super::cache::CacheDir; 16 17 use anyhow::{Error, Result}; 18 19 /// # Git源 20 /// 21 /// 从Git仓库获取源码 22 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 23 pub struct GitSource { 24 /// Git仓库地址 25 url: String, 26 /// 分支（可选，如果为空，则拉取master）branch和revision只能同时指定一个 27 branch: Option<String>, 28 /// 特定的提交的hash值（可选，如果为空，则拉取branch的最新提交） 29 revision: Option<String>, 30 } 31 32 impl GitSource { 33 pub fn new(url: String, branch: Option<String>, revision: Option<String>) -> Self { 34 Self { 35 url, 36 branch, 37 revision, 38 } 39 } 40 /// # 验证参数合法性 41 /// 42 /// 仅进行形式校验，不会检查Git仓库是否存在，以及分支是否存在、是否有权限访问等 43 pub fn validate(&mut self) -> Result<()> { 44 if self.url.is_empty() { 45 return Err(Error::msg("url is empty")); 46 } 47 // branch和revision不能同时为空 48 if self.branch.is_none() && self.revision.is_none() { 49 self.branch = Some("master".to_string()); 50 } 51 // branch和revision只能同时指定一个 52 if self.branch.is_some() && self.revision.is_some() { 53 return Err(Error::msg("branch and revision are both specified")); 54 } 55 56 if self.branch.is_some() { 57 if self.branch.as_ref().unwrap().is_empty() { 58 return Err(Error::msg("branch is empty")); 59 } 60 } 61 if self.revision.is_some() { 62 if self.revision.as_ref().unwrap().is_empty() { 63 return Err(Error::msg("revision is empty")); 64 } 65 } 66 return Ok(()); 67 } 68 69 pub fn trim(&mut self) { 70 self.url = self.url.trim().to_string(); 71 if let Some(branch) = &mut self.branch { 72 *branch = branch.trim().to_string(); 73 } 74 75 if let Some(revision) = &mut self.revision { 76 *revision = revision.trim().to_string(); 77 } 78 } 79 80 /// # 确保Git仓库已经克隆到指定目录，并且切换到指定分支/Revision 81 /// 82 /// 如果目录不存在，则会自动创建 83 /// 84 /// ## 参数 85 /// 86 /// - `target_dir` - 目标目录 87 /// 88 /// ## 返回 89 /// 90 /// - `Ok(())` - 成功 91 /// - `Err(String)` - 失败，错误信息 92 pub fn prepare(&self, target_dir: &CacheDir) -> Result<(), String> { 93 info!( 94 "Preparing git repo: {}, branch: {:?}, revision: {:?}", 95 self.url, self.branch, self.revision 96 ); 97 98 target_dir.create().map_err(|e| { 99 format!( 100 "Failed to create target dir: {}, message: {e:?}", 101 target_dir.path.display() 102 ) 103 })?; 104 105 if target_dir.is_empty().map_err(|e| { 106 format!( 107 "Failed to check if target dir is empty: {}, message: {e:?}", 108 target_dir.path.display() 109 ) 110 })? { 111 info!("Target dir is empty, cloning repo"); 112 self.clone_repo(target_dir)?; 113 } 114 115 self.checkout(target_dir)?; 116 117 self.pull(target_dir)?; 118 119 return Ok(()); 120 } 121 122 fn check_repo(&self, target_dir: &CacheDir) -> Result<bool, String> { 123 let path: &PathBuf = &target_dir.path; 124 let mut cmd = Command::new("git"); 125 cmd.arg("remote").arg("get-url").arg("origin"); 126 127 // 设置工作目录 128 cmd.current_dir(path); 129 130 // 创建子进程，执行命令 131 let proc: std::process::Child = cmd 132 .stderr(Stdio::piped()) 133 .stdout(Stdio::piped()) 134 .spawn() 135 .map_err(|e| e.to_string())?; 136 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 137 138 if output.status.success() { 139 let mut r = String::from_utf8(output.stdout).unwrap(); 140 r.pop(); 141 Ok(r == self.url) 142 } else { 143 return Err(format!( 144 "git remote get-url origin failed, status: {:?}, stderr: {:?}", 145 output.status, 146 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 147 )); 148 } 149 } 150 151 fn set_url(&self, target_dir: &CacheDir) -> Result<(), String> { 152 let path: &PathBuf = &target_dir.path; 153 let mut cmd = Command::new("git"); 154 cmd.arg("remote") 155 .arg("set-url") 156 .arg("origin") 157 .arg(self.url.as_str()); 158 159 // 设置工作目录 160 cmd.current_dir(path); 161 162 // 创建子进程，执行命令 163 let proc: std::process::Child = cmd 164 .stderr(Stdio::piped()) 165 .spawn() 166 .map_err(|e| e.to_string())?; 167 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 168 169 if !output.status.success() { 170 return Err(format!( 171 "git remote set-url origin failed, status: {:?}, stderr: {:?}", 172 output.status, 173 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 174 )); 175 } 176 Ok(()) 177 } 178 179 fn checkout(&self, target_dir: &CacheDir) -> Result<(), String> { 180 // 确保目标目录中的仓库为所指定仓库 181 if !self.check_repo(target_dir).map_err(|e| { 182 format!( 183 "Failed to check repo: {}, message: {e:?}", 184 target_dir.path.display() 185 ) 186 })? { 187 info!("Target dir isn't specified repo, change remote url"); 188 self.set_url(target_dir)?; 189 } 190 191 let do_checkout = || -> Result<(), String> { 192 let mut cmd = Command::new("git"); 193 cmd.current_dir(&target_dir.path); 194 cmd.arg("checkout"); 195 196 if let Some(branch) = &self.branch { 197 cmd.arg(branch); 198 } 199 if let Some(revision) = &self.revision { 200 cmd.arg(revision); 201 } 202 203 // 强制切换分支，且安静模式 204 cmd.arg("-f").arg("-q"); 205 206 // 创建子进程，执行命令 207 let proc: std::process::Child = cmd 208 .stderr(Stdio::piped()) 209 .spawn() 210 .map_err(|e| e.to_string())?; 211 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 212 213 if !output.status.success() { 214 return Err(format!( 215 "Failed to checkout {}, message: {}", 216 target_dir.path.display(), 217 String::from_utf8_lossy(&output.stdout) 218 )); 219 } 220 221 let mut subcmd = Command::new("git"); 222 subcmd.current_dir(&target_dir.path); 223 subcmd.arg("submodule").arg("update").arg("--remote"); 224 225 //当checkout仓库的子进程结束后，启动checkout子模块的子进程 226 let subproc: std::process::Child = subcmd 227 .stderr(Stdio::piped()) 228 .spawn() 229 .map_err(|e| e.to_string())?; 230 let suboutput = subproc.wait_with_output().map_err(|e| e.to_string())?; 231 232 if !suboutput.status.success() { 233 return Err(format!( 234 "Failed to checkout submodule {}, message: {}", 235 target_dir.path.display(), 236 String::from_utf8_lossy(&suboutput.stdout) 237 )); 238 } 239 return Ok(()); 240 }; 241 242 if let Err(_) = do_checkout() { 243 // 如果切换分支失败，则尝试重新fetch 244 if self.revision.is_some() { 245 self.set_fetch_config(target_dir)?; 246 self.unshallow(target_dir)? 247 }; 248 249 self.fetch_all(target_dir).ok(); 250 do_checkout()?; 251 } 252 253 return Ok(()); 254 } 255 256 pub fn clone_repo(&self, cache_dir: &CacheDir) -> Result<(), String> { 257 let path: &PathBuf = &cache_dir.path; 258 let mut cmd = Command::new("git"); 259 cmd.arg("clone").arg(&self.url).arg(".").arg("--recursive"); 260 261 if let Some(branch) = &self.branch { 262 cmd.arg("--branch").arg(branch).arg("--depth").arg("1"); 263 } 264 265 // 对于克隆，如果指定了revision，则直接克隆整个仓库，稍后再切换到指定的revision 266 267 // 设置工作目录 268 cmd.current_dir(path); 269 270 // 创建子进程，执行命令 271 let proc: std::process::Child = cmd 272 .stderr(Stdio::piped()) 273 .stdout(Stdio::inherit()) 274 .spawn() 275 .map_err(|e| e.to_string())?; 276 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 277 278 if !output.status.success() { 279 return Err(format!( 280 "clone git repo failed, status: {:?}, stderr: {:?}", 281 output.status, 282 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 283 )); 284 } 285 286 let mut subcmd = Command::new("git"); 287 subcmd 288 .arg("submodule") 289 .arg("update") 290 .arg("--init") 291 .arg("--recursive") 292 .arg("--force"); 293 294 subcmd.current_dir(path); 295 296 //当克隆仓库的子进程结束后，启动保证克隆子模块的子进程 297 let subproc: std::process::Child = subcmd 298 .stderr(Stdio::piped()) 299 .stdout(Stdio::inherit()) 300 .spawn() 301 .map_err(|e| e.to_string())?; 302 let suboutput = subproc.wait_with_output().map_err(|e| e.to_string())?; 303 304 if !suboutput.status.success() { 305 return Err(format!( 306 "clone submodule failed, status: {:?}, stderr: {:?}", 307 suboutput.status, 308 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&suboutput.stderr), 5) 309 )); 310 } 311 return Ok(()); 312 } 313 314 /// 设置fetch所有分支 315 fn set_fetch_config(&self, target_dir: &CacheDir) -> Result<(), String> { 316 let mut cmd = Command::new("git"); 317 cmd.current_dir(&target_dir.path); 318 cmd.arg("config") 319 .arg("remote.origin.fetch") 320 .arg("+refs/heads/*:refs/remotes/origin/*"); 321 322 // 创建子进程，执行命令 323 let proc: std::process::Child = cmd 324 .stderr(Stdio::piped()) 325 .spawn() 326 .map_err(|e| e.to_string())?; 327 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 328 329 if !output.status.success() { 330 return Err(format!( 331 "Failed to set fetch config {}, message: {}", 332 target_dir.path.display(), 333 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 334 )); 335 } 336 return Ok(()); 337 } 338 /// # 把浅克隆的仓库变成深克隆 339 fn unshallow(&self, target_dir: &CacheDir) -> Result<(), String> { 340 if self.is_shallow(target_dir)? == false { 341 return Ok(()); 342 } 343 344 let mut cmd = Command::new("git"); 345 cmd.current_dir(&target_dir.path); 346 cmd.arg("fetch").arg("--unshallow"); 347 348 cmd.arg("-f"); 349 350 // 创建子进程，执行命令 351 let proc: std::process::Child = cmd 352 .stderr(Stdio::piped()) 353 .spawn() 354 .map_err(|e| e.to_string())?; 355 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 356 357 if !output.status.success() { 358 return Err(format!( 359 "Failed to unshallow {}, message: {}", 360 target_dir.path.display(), 361 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 362 )); 363 } 364 return Ok(()); 365 } 366 367 /// 判断当前仓库是否是浅克隆 368 fn is_shallow(&self, target_dir: &CacheDir) -> Result<bool, String> { 369 let mut cmd = Command::new("git"); 370 cmd.current_dir(&target_dir.path); 371 cmd.arg("rev-parse").arg("--is-shallow-repository"); 372 373 let proc: std::process::Child = cmd 374 .stderr(Stdio::piped()) 375 .spawn() 376 .map_err(|e| e.to_string())?; 377 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 378 379 if !output.status.success() { 380 return Err(format!( 381 "Failed to check if shallow {}, message: {}", 382 target_dir.path.display(), 383 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 384 )); 385 } 386 387 let is_shallow = String::from_utf8_lossy(&output.stdout).trim() == "true"; 388 return Ok(is_shallow); 389 } 390 391 fn fetch_all(&self, target_dir: &CacheDir) -> Result<(), String> { 392 self.set_fetch_config(target_dir)?; 393 let mut cmd = Command::new("git"); 394 cmd.current_dir(&target_dir.path); 395 cmd.arg("fetch").arg("--all"); 396 397 // 安静模式 398 cmd.arg("-f").arg("-q"); 399 400 // 创建子进程，执行命令 401 let proc: std::process::Child = cmd 402 .stderr(Stdio::piped()) 403 .spawn() 404 .map_err(|e| e.to_string())?; 405 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 406 407 if !output.status.success() { 408 return Err(format!( 409 "Failed to fetch all {}, message: {}", 410 target_dir.path.display(), 411 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 412 )); 413 } 414 415 return Ok(()); 416 } 417 418 fn pull(&self, target_dir: &CacheDir) -> Result<(), String> { 419 // 如果没有指定branch，则不执行pull 420 if !self.branch.is_some() { 421 return Ok(()); 422 } 423 info!("git pulling: {}", target_dir.path.display()); 424 425 let mut cmd = Command::new("git"); 426 cmd.current_dir(&target_dir.path); 427 cmd.arg("pull"); 428 429 // 安静模式 430 cmd.arg("-f").arg("-q"); 431 432 // 创建子进程，执行命令 433 let proc: std::process::Child = cmd 434 .stderr(Stdio::piped()) 435 .spawn() 436 .map_err(|e| e.to_string())?; 437 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 438 439 // 如果pull失败，且指定了branch，则报错 440 if !output.status.success() { 441 return Err(format!( 442 "Failed to pull {}, message: {}", 443 target_dir.path.display(), 444 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 445 )); 446 } 447 448 return Ok(()); 449 } 450 } 451 452 /// # 本地源 453 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 454 pub struct LocalSource { 455 /// 本地目录/文件的路径 456 path: PathBuf, 457 } 458 459 impl LocalSource { 460 #[allow(dead_code)] 461 pub fn new(path: PathBuf) -> Self { 462 Self { path } 463 } 464 465 pub fn validate(&self, expect_file: Option<bool>) -> Result<()> { 466 if !self.path.exists() { 467 return Err(Error::msg(format!("path {:?} not exists", self.path))); 468 } 469 470 if let Some(expect_file) = expect_file { 471 if expect_file && !self.path.is_file() { 472 return Err(Error::msg(format!("path {:?} is not a file", self.path))); 473 } 474 475 if !expect_file && !self.path.is_dir() { 476 return Err(Error::msg(format!( 477 "path {:?} is not a directory", 478 self.path 479 ))); 480 } 481 } 482 483 return Ok(()); 484 } 485 486 pub fn trim(&mut self) {} 487 488 pub fn path(&self) -> &PathBuf { 489 &self.path 490 } 491 } 492 493 /// # 在线压缩包源 494 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 495 pub struct ArchiveSource { 496 /// 压缩包的URL 497 url: String, 498 } 499 500 impl ArchiveSource { 501 #[allow(dead_code)] 502 pub fn new(url: String) -> Self { 503 Self { url } 504 } 505 pub fn validate(&self) -> Result<()> { 506 if self.url.is_empty() { 507 return Err(Error::msg("url is empty")); 508 } 509 510 // 判断是一个网址 511 if let Ok(url) = Url::parse(&self.url) { 512 if url.scheme() != "http" && url.scheme() != "https" { 513 return Err(Error::msg(format!( 514 "url {:?} is not a http/https url", 515 self.url 516 ))); 517 } 518 } else { 519 return Err(Error::msg(format!("url {:?} is not a valid url", self.url))); 520 } 521 return Ok(()); 522 } 523 524 pub fn trim(&mut self) { 525 self.url = self.url.trim().to_string(); 526 } 527 528 /// @brief 下载压缩包并把其中的文件提取至target_dir目录下 529 /// 530 ///从URL中下载压缩包到临时文件夹 target_dir/DRAGONOS_ARCHIVE_TEMP 后 531 ///原地解压，提取文件后删除下载的压缩包。如果 target_dir 非空，就直接使用 532 ///其中内容，不进行重复下载和覆盖 533 /// 534 /// @param target_dir 文件缓存目录 535 /// 536 /// @return 根据结果返回OK或Err 537 pub fn download_unzip(&self, target_dir: &CacheDir) -> Result<(), String> { 538 let url = Url::parse(&self.url).unwrap(); 539 let archive_name = url.path_segments().unwrap().last().unwrap(); 540 let path = &(target_dir.path.join("DRAGONOS_ARCHIVE_TEMP")); 541 //如果source目录没有临时文件夹，且不为空，说明之前成功执行过一次，那么就直接使用之前的缓存 542 if !path.exists() 543 && !target_dir.is_empty().map_err(|e| { 544 format!( 545 "Failed to check if target dir is empty: {}, message: {e:?}", 546 target_dir.path.display() 547 ) 548 })? 549 { 550 //如果source文件夹非空，就直接使用，不再重复下载压缩文件，这里可以考虑加入交互 551 info!("Source files already exist. Using previous source file cache. You should clean {:?} before re-download the archive ", target_dir.path); 552 return Ok(()); 553 } 554 555 if path.exists() { 556 std::fs::remove_dir_all(path).map_err(|e| e.to_string())?; 557 } 558 //创建临时目录 559 std::fs::create_dir(path).map_err(|e| e.to_string())?; 560 info!("downloading {:?}", archive_name); 561 FileUtils::download_file(&self.url, path).map_err(|e| e.to_string())?; 562 //下载成功，开始尝试解压 563 info!("download {:?} finished, start unzip", archive_name); 564 let archive_file = ArchiveFile::new(&path.join(archive_name)); 565 archive_file.unzip()?; 566 //删除创建的临时文件夹 567 std::fs::remove_dir_all(path).map_err(|e| e.to_string())?; 568 return Ok(()); 569 } 570 } 571 572 pub struct ArchiveFile { 573 archive_path: PathBuf, 574 archive_name: String, 575 archive_type: ArchiveType, 576 } 577 578 impl ArchiveFile { 579 pub fn new(archive_path: &PathBuf) -> Self { 580 info!("archive_path: {:?}", archive_path); 581 //匹配压缩文件类型 582 let archive_name = archive_path.file_name().unwrap().to_str().unwrap(); 583 for (regex, archivetype) in [ 584 (Regex::new(r"^(.+)\.tar\.gz$").unwrap(), ArchiveType::TarGz), 585 (Regex::new(r"^(.+)\.tar\.xz$").unwrap(), ArchiveType::TarXz), 586 (Regex::new(r"^(.+)\.zip$").unwrap(), ArchiveType::Zip), 587 ] { 588 if regex.is_match(archive_name) { 589 return Self { 590 archive_path: archive_path.parent().unwrap().to_path_buf(), 591 archive_name: archive_name.to_string(), 592 archive_type: archivetype, 593 }; 594 } 595 } 596 Self { 597 archive_path: archive_path.parent().unwrap().to_path_buf(), 598 archive_name: archive_name.to_string(), 599 archive_type: ArchiveType::Undefined, 600 } 601 } 602 603 /// @brief 对self.archive_path路径下名为self.archive_name的压缩文件(tar.gz或zip)进行解压缩 604 /// 605 /// 在此函数中进行路径和文件名有效性的判断，如果有效的话就开始解压缩，根据ArchiveType枚举类型来 606 /// 生成不同的命令来对压缩文件进行解压缩，暂时只支持tar.gz和zip格式，并且都是通过调用bash来解压缩 607 /// 没有引入第三方rust库 608 /// 609 /// 610 /// @return 根据结果返回OK或Err 611 612 pub fn unzip(&self) -> Result<(), String> { 613 let path = &self.archive_path; 614 if !path.is_dir() { 615 return Err(format!("Archive directory {:?} is wrong", path)); 616 } 617 if !path.join(&self.archive_name).is_file() { 618 return Err(format!( 619 " {:?} is not a file", 620 path.join(&self.archive_name) 621 )); 622 } 623 //根据压缩文件的类型生成cmd指令 624 match &self.archive_type { 625 ArchiveType::TarGz | ArchiveType::TarXz => { 626 let mut cmd = Command::new("tar"); 627 cmd.arg("-xf").arg(&self.archive_name); 628 let proc: std::process::Child = cmd 629 .current_dir(path) 630 .stderr(Stdio::piped()) 631 .stdout(Stdio::inherit()) 632 .spawn() 633 .map_err(|e| e.to_string())?; 634 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 635 if !output.status.success() { 636 return Err(format!( 637 "unzip failed, status: {:?}, stderr: {:?}", 638 output.status, 639 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 640 )); 641 } 642 } 643 644 ArchiveType::Zip => { 645 let file = File::open(&self.archive_path.join(&self.archive_name)) 646 .map_err(|e| e.to_string())?; 647 let mut archive = ZipArchive::new(file).map_err(|e| e.to_string())?; 648 for i in 0..archive.len() { 649 let mut file = archive.by_index(i).map_err(|e| e.to_string())?; 650 let outpath = match file.enclosed_name() { 651 Some(path) => self.archive_path.join(path), 652 None => continue, 653 }; 654 if (*file.name()).ends_with('/') { 655 std::fs::create_dir_all(&outpath).map_err(|e| e.to_string())?; 656 } else { 657 if let Some(p) = outpath.parent() { 658 if !p.exists() { 659 std::fs::create_dir_all(&p).map_err(|e| e.to_string())?; 660 } 661 } 662 let mut outfile = File::create(&outpath).map_err(|e| e.to_string())?; 663 std::io::copy(&mut file, &mut outfile).map_err(|e| e.to_string())?; 664 } 665 //设置解压后权限，在Linux中Unzip会丢失权限 666 #[cfg(unix)] 667 { 668 if let Some(mode) = file.unix_mode() { 669 std::fs::set_permissions( 670 &outpath, 671 std::fs::Permissions::from_mode(mode), 672 ) 673 .map_err(|e| e.to_string())?; 674 } 675 } 676 } 677 } 678 _ => { 679 return Err("unsupported archive type".to_string()); 680 } 681 } 682 //删除下载的压缩包 683 info!("unzip successfully, removing archive "); 684 std::fs::remove_file(path.join(&self.archive_name)).map_err(|e| e.to_string())?; 685 //从解压的文件夹中提取出文件并删除下载的压缩包等价于指令"cd *;mv ./* ../../" 686 for entry in path.read_dir().map_err(|e| e.to_string())? { 687 let entry = entry.map_err(|e| e.to_string())?; 688 let path = entry.path(); 689 FileUtils::move_files(&path, &self.archive_path.parent().unwrap()) 690 .map_err(|e| e.to_string())?; 691 //删除空的单独文件夹 692 std::fs::remove_dir_all(&path).map_err(|e| e.to_string())?; 693 } 694 return Ok(()); 695 } 696 } 697 698 pub enum ArchiveType { 699 TarGz, 700 TarXz, 701 Zip, 702 Undefined, 703 } 704