1 use log::info; 2 use regex::Regex; 3 use reqwest::Url; 4 use serde::{Deserialize, Serialize}; 5 use std::os::unix::fs::PermissionsExt; 6 use std::{ 7 fs::File, 8 path::PathBuf, 9 process::{Command, Stdio}, 10 }; 11 use zip::ZipArchive; 12 13 use crate::utils::{file::FileUtils, stdio::StdioUtils}; 14 15 use super::cache::CacheDir; 16 17 /// # Git源 18 /// 19 /// 从Git仓库获取源码 20 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 21 pub struct GitSource { 22 /// Git仓库地址 23 url: String, 24 /// 分支（可选，如果为空，则拉取master）branch和revision只能同时指定一个 25 branch: Option<String>, 26 /// 特定的提交的hash值（可选，如果为空，则拉取branch的最新提交） 27 revision: Option<String>, 28 } 29 30 impl GitSource { 31 pub fn new(url: String, branch: Option<String>, revision: Option<String>) -> Self { 32 Self { 33 url, 34 branch, 35 revision, 36 } 37 } 38 /// # 验证参数合法性 39 /// 40 /// 仅进行形式校验，不会检查Git仓库是否存在，以及分支是否存在、是否有权限访问等 41 pub fn validate(&mut self) -> Result<(), String> { 42 if self.url.is_empty() { 43 return Err("url is empty".to_string()); 44 } 45 // branch和revision不能同时为空 46 if self.branch.is_none() && self.revision.is_none() { 47 self.branch = Some("master".to_string()); 48 } 49 // branch和revision只能同时指定一个 50 if self.branch.is_some() && self.revision.is_some() { 51 return Err("branch and revision are both specified".to_string()); 52 } 53 54 if self.branch.is_some() { 55 if self.branch.as_ref().unwrap().is_empty() { 56 return Err("branch is empty".to_string()); 57 } 58 } 59 if self.revision.is_some() { 60 if self.revision.as_ref().unwrap().is_empty() { 61 return Err("revision is empty".to_string()); 62 } 63 } 64 return Ok(()); 65 } 66 67 pub fn trim(&mut self) { 68 self.url = self.url.trim().to_string(); 69 if let Some(branch) = &mut self.branch { 70 *branch = branch.trim().to_string(); 71 } 72 73 if let Some(revision) = &mut self.revision { 74 *revision = revision.trim().to_string(); 75 } 76 } 77 78 /// # 确保Git仓库已经克隆到指定目录，并且切换到指定分支/Revision 79 /// 80 /// 如果目录不存在，则会自动创建 81 /// 82 /// ## 参数 83 /// 84 /// - `target_dir` - 目标目录 85 /// 86 /// ## 返回 87 /// 88 /// - `Ok(())` - 成功 89 /// - `Err(String)` - 失败，错误信息 90 pub fn prepare(&self, target_dir: &CacheDir) -> Result<(), String> { 91 info!( 92 "Preparing git repo: {}, branch: {:?}, revision: {:?}", 93 self.url, self.branch, self.revision 94 ); 95 96 target_dir.create().map_err(|e| { 97 format!( 98 "Failed to create target dir: {}, message: {e:?}", 99 target_dir.path.display() 100 ) 101 })?; 102 103 if target_dir.is_empty().map_err(|e| { 104 format!( 105 "Failed to check if target dir is empty: {}, message: {e:?}", 106 target_dir.path.display() 107 ) 108 })? { 109 info!("Target dir is empty, cloning repo"); 110 self.clone_repo(target_dir)?; 111 } 112 113 self.checkout(target_dir)?; 114 115 self.pull(target_dir)?; 116 117 return Ok(()); 118 } 119 120 fn check_repo(&self, target_dir: &CacheDir) -> Result<bool, String> { 121 let path: &PathBuf = &target_dir.path; 122 let mut cmd = Command::new("git"); 123 cmd.arg("remote").arg("get-url").arg("origin"); 124 125 // 设置工作目录 126 cmd.current_dir(path); 127 128 // 创建子进程，执行命令 129 let proc: std::process::Child = cmd 130 .stderr(Stdio::piped()) 131 .stdout(Stdio::piped()) 132 .spawn() 133 .map_err(|e| e.to_string())?; 134 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 135 136 if output.status.success() { 137 let mut r = String::from_utf8(output.stdout).unwrap(); 138 r.pop(); 139 Ok(r == self.url) 140 } else { 141 return Err(format!( 142 "git remote get-url origin failed, status: {:?}, stderr: {:?}", 143 output.status, 144 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 145 )); 146 } 147 } 148 149 fn set_url(&self, target_dir: &CacheDir) -> Result<(), String> { 150 let path: &PathBuf = &target_dir.path; 151 let mut cmd = Command::new("git"); 152 cmd.arg("remote") 153 .arg("set-url") 154 .arg("origin") 155 .arg(self.url.as_str()); 156 157 // 设置工作目录 158 cmd.current_dir(path); 159 160 // 创建子进程，执行命令 161 let proc: std::process::Child = cmd 162 .stderr(Stdio::piped()) 163 .spawn() 164 .map_err(|e| e.to_string())?; 165 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 166 167 if !output.status.success() { 168 return Err(format!( 169 "git remote set-url origin failed, status: {:?}, stderr: {:?}", 170 output.status, 171 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 172 )); 173 } 174 Ok(()) 175 } 176 177 fn checkout(&self, target_dir: &CacheDir) -> Result<(), String> { 178 // 确保目标目录中的仓库为所指定仓库 179 if !self.check_repo(target_dir).map_err(|e| { 180 format!( 181 "Failed to check repo: {}, message: {e:?}", 182 target_dir.path.display() 183 ) 184 })? { 185 info!("Target dir isn't specified repo, change remote url"); 186 self.set_url(target_dir)?; 187 } 188 189 let do_checkout = || -> Result<(), String> { 190 let mut cmd = Command::new("git"); 191 cmd.current_dir(&target_dir.path); 192 cmd.arg("checkout"); 193 194 if let Some(branch) = &self.branch { 195 cmd.arg(branch); 196 } 197 if let Some(revision) = &self.revision { 198 cmd.arg(revision); 199 } 200 201 // 强制切换分支，且安静模式 202 cmd.arg("-f").arg("-q"); 203 204 // 创建子进程，执行命令 205 let proc: std::process::Child = cmd 206 .stderr(Stdio::piped()) 207 .spawn() 208 .map_err(|e| e.to_string())?; 209 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 210 211 if !output.status.success() { 212 return Err(format!( 213 "Failed to checkout {}, message: {}", 214 target_dir.path.display(), 215 String::from_utf8_lossy(&output.stdout) 216 )); 217 } 218 219 let mut subcmd = Command::new("git"); 220 subcmd.current_dir(&target_dir.path); 221 subcmd.arg("submodule").arg("update").arg("--remote"); 222 223 //当checkout仓库的子进程结束后，启动checkout子模块的子进程 224 let subproc: std::process::Child = subcmd 225 .stderr(Stdio::piped()) 226 .spawn() 227 .map_err(|e| e.to_string())?; 228 let suboutput = subproc.wait_with_output().map_err(|e| e.to_string())?; 229 230 if !suboutput.status.success() { 231 return Err(format!( 232 "Failed to checkout submodule {}, message: {}", 233 target_dir.path.display(), 234 String::from_utf8_lossy(&suboutput.stdout) 235 )); 236 } 237 return Ok(()); 238 }; 239 240 if let Err(_) = do_checkout() { 241 // 如果切换分支失败，则尝试重新fetch 242 if self.revision.is_some() { 243 self.set_fetch_config(target_dir)?; 244 self.unshallow(target_dir)? 245 }; 246 247 self.fetch_all(target_dir).ok(); 248 do_checkout()?; 249 } 250 251 return Ok(()); 252 } 253 254 pub fn clone_repo(&self, cache_dir: &CacheDir) -> Result<(), String> { 255 let path: &PathBuf = &cache_dir.path; 256 let mut cmd = Command::new("git"); 257 cmd.arg("clone").arg(&self.url).arg(".").arg("--recursive"); 258 259 if let Some(branch) = &self.branch { 260 cmd.arg("--branch").arg(branch).arg("--depth").arg("1"); 261 } 262 263 // 对于克隆，如果指定了revision，则直接克隆整个仓库，稍后再切换到指定的revision 264 265 // 设置工作目录 266 cmd.current_dir(path); 267 268 // 创建子进程，执行命令 269 let proc: std::process::Child = cmd 270 .stderr(Stdio::piped()) 271 .stdout(Stdio::inherit()) 272 .spawn() 273 .map_err(|e| e.to_string())?; 274 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 275 276 if !output.status.success() { 277 return Err(format!( 278 "clone git repo failed, status: {:?}, stderr: {:?}", 279 output.status, 280 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 281 )); 282 } 283 284 let mut subcmd = Command::new("git"); 285 subcmd 286 .arg("submodule") 287 .arg("update") 288 .arg("--init") 289 .arg("--recursive") 290 .arg("--force"); 291 292 subcmd.current_dir(path); 293 294 //当克隆仓库的子进程结束后，启动保证克隆子模块的子进程 295 let subproc: std::process::Child = subcmd 296 .stderr(Stdio::piped()) 297 .stdout(Stdio::inherit()) 298 .spawn() 299 .map_err(|e| e.to_string())?; 300 let suboutput = subproc.wait_with_output().map_err(|e| e.to_string())?; 301 302 if !suboutput.status.success() { 303 return Err(format!( 304 "clone submodule failed, status: {:?}, stderr: {:?}", 305 suboutput.status, 306 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&suboutput.stderr), 5) 307 )); 308 } 309 return Ok(()); 310 } 311 312 /// 设置fetch所有分支 313 fn set_fetch_config(&self, target_dir: &CacheDir) -> Result<(), String> { 314 let mut cmd = Command::new("git"); 315 cmd.current_dir(&target_dir.path); 316 cmd.arg("config") 317 .arg("remote.origin.fetch") 318 .arg("+refs/heads/*:refs/remotes/origin/*"); 319 320 // 创建子进程，执行命令 321 let proc: std::process::Child = cmd 322 .stderr(Stdio::piped()) 323 .spawn() 324 .map_err(|e| e.to_string())?; 325 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 326 327 if !output.status.success() { 328 return Err(format!( 329 "Failed to set fetch config {}, message: {}", 330 target_dir.path.display(), 331 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 332 )); 333 } 334 return Ok(()); 335 } 336 /// # 把浅克隆的仓库变成深克隆 337 fn unshallow(&self, target_dir: &CacheDir) -> Result<(), String> { 338 if self.is_shallow(target_dir)? == false { 339 return Ok(()); 340 } 341 342 let mut cmd = Command::new("git"); 343 cmd.current_dir(&target_dir.path); 344 cmd.arg("fetch").arg("--unshallow"); 345 346 cmd.arg("-f"); 347 348 // 创建子进程，执行命令 349 let proc: std::process::Child = cmd 350 .stderr(Stdio::piped()) 351 .spawn() 352 .map_err(|e| e.to_string())?; 353 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 354 355 if !output.status.success() { 356 return Err(format!( 357 "Failed to unshallow {}, message: {}", 358 target_dir.path.display(), 359 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 360 )); 361 } 362 return Ok(()); 363 } 364 365 /// 判断当前仓库是否是浅克隆 366 fn is_shallow(&self, target_dir: &CacheDir) -> Result<bool, String> { 367 let mut cmd = Command::new("git"); 368 cmd.current_dir(&target_dir.path); 369 cmd.arg("rev-parse").arg("--is-shallow-repository"); 370 371 let proc: std::process::Child = cmd 372 .stderr(Stdio::piped()) 373 .spawn() 374 .map_err(|e| e.to_string())?; 375 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 376 377 if !output.status.success() { 378 return Err(format!( 379 "Failed to check if shallow {}, message: {}", 380 target_dir.path.display(), 381 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 382 )); 383 } 384 385 let is_shallow = String::from_utf8_lossy(&output.stdout).trim() == "true"; 386 return Ok(is_shallow); 387 } 388 389 fn fetch_all(&self, target_dir: &CacheDir) -> Result<(), String> { 390 self.set_fetch_config(target_dir)?; 391 let mut cmd = Command::new("git"); 392 cmd.current_dir(&target_dir.path); 393 cmd.arg("fetch").arg("--all"); 394 395 // 安静模式 396 cmd.arg("-f").arg("-q"); 397 398 // 创建子进程，执行命令 399 let proc: std::process::Child = cmd 400 .stderr(Stdio::piped()) 401 .spawn() 402 .map_err(|e| e.to_string())?; 403 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 404 405 if !output.status.success() { 406 return Err(format!( 407 "Failed to fetch all {}, message: {}", 408 target_dir.path.display(), 409 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 410 )); 411 } 412 413 return Ok(()); 414 } 415 416 fn pull(&self, target_dir: &CacheDir) -> Result<(), String> { 417 // 如果没有指定branch，则不执行pull 418 if !self.branch.is_some() { 419 return Ok(()); 420 } 421 info!("git pulling: {}", target_dir.path.display()); 422 423 let mut cmd = Command::new("git"); 424 cmd.current_dir(&target_dir.path); 425 cmd.arg("pull"); 426 427 // 安静模式 428 cmd.arg("-f").arg("-q"); 429 430 // 创建子进程，执行命令 431 let proc: std::process::Child = cmd 432 .stderr(Stdio::piped()) 433 .spawn() 434 .map_err(|e| e.to_string())?; 435 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 436 437 // 如果pull失败，且指定了branch，则报错 438 if !output.status.success() { 439 return Err(format!( 440 "Failed to pull {}, message: {}", 441 target_dir.path.display(), 442 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 443 )); 444 } 445 446 return Ok(()); 447 } 448 } 449 450 /// # 本地源 451 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 452 pub struct LocalSource { 453 /// 本地目录/文件的路径 454 path: PathBuf, 455 } 456 457 impl LocalSource { 458 #[allow(dead_code)] 459 pub fn new(path: PathBuf) -> Self { 460 Self { path } 461 } 462 463 pub fn validate(&self, expect_file: Option<bool>) -> Result<(), String> { 464 if !self.path.exists() { 465 return Err(format!("path {:?} not exists", self.path)); 466 } 467 468 if let Some(expect_file) = expect_file { 469 if expect_file && !self.path.is_file() { 470 return Err(format!("path {:?} is not a file", self.path)); 471 } 472 473 if !expect_file && !self.path.is_dir() { 474 return Err(format!("path {:?} is not a directory", self.path)); 475 } 476 } 477 478 return Ok(()); 479 } 480 481 pub fn trim(&mut self) {} 482 483 pub fn path(&self) -> &PathBuf { 484 &self.path 485 } 486 } 487 488 /// # 在线压缩包源 489 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 490 pub struct ArchiveSource { 491 /// 压缩包的URL 492 url: String, 493 } 494 495 impl ArchiveSource { 496 #[allow(dead_code)] 497 pub fn new(url: String) -> Self { 498 Self { url } 499 } 500 pub fn validate(&self) -> Result<(), String> { 501 if self.url.is_empty() { 502 return Err("url is empty".to_string()); 503 } 504 505 // 判断是一个网址 506 if let Ok(url) = Url::parse(&self.url) { 507 if url.scheme() != "http" && url.scheme() != "https" { 508 return Err(format!("url {:?} is not a http/https url", self.url)); 509 } 510 } else { 511 return Err(format!("url {:?} is not a valid url", self.url)); 512 } 513 return Ok(()); 514 } 515 516 pub fn trim(&mut self) { 517 self.url = self.url.trim().to_string(); 518 } 519 520 /// @brief 下载压缩包并把其中的文件提取至target_dir目录下 521 /// 522 ///从URL中下载压缩包到临时文件夹 target_dir/DRAGONOS_ARCHIVE_TEMP 后 523 ///原地解压，提取文件后删除下载的压缩包。如果 target_dir 非空，就直接使用 524 ///其中内容，不进行重复下载和覆盖 525 /// 526 /// @param target_dir 文件缓存目录 527 /// 528 /// @return 根据结果返回OK或Err 529 pub fn download_unzip(&self, target_dir: &CacheDir) -> Result<(), String> { 530 let url = Url::parse(&self.url).unwrap(); 531 let archive_name = url.path_segments().unwrap().last().unwrap(); 532 let path = &(target_dir.path.join("DRAGONOS_ARCHIVE_TEMP")); 533 //如果source目录没有临时文件夹，且不为空，说明之前成功执行过一次，那么就直接使用之前的缓存 534 if !path.exists() 535 && !target_dir.is_empty().map_err(|e| { 536 format!( 537 "Failed to check if target dir is empty: {}, message: {e:?}", 538 target_dir.path.display() 539 ) 540 })? 541 { 542 //如果source文件夹非空，就直接使用，不再重复下载压缩文件，这里可以考虑加入交互 543 info!("Source files already exist. Using previous source file cache. You should clean {:?} before re-download the archive ", target_dir.path); 544 return Ok(()); 545 } 546 547 if path.exists() { 548 std::fs::remove_dir_all(path).map_err(|e| e.to_string())?; 549 } 550 //创建临时目录 551 std::fs::create_dir(path).map_err(|e| e.to_string())?; 552 info!("downloading {:?}", archive_name); 553 FileUtils::download_file(&self.url, path).map_err(|e| e.to_string())?; 554 //下载成功，开始尝试解压 555 info!("download {:?} finished, start unzip", archive_name); 556 let archive_file = ArchiveFile::new(&path.join(archive_name)); 557 archive_file.unzip()?; 558 //删除创建的临时文件夹 559 std::fs::remove_dir_all(path).map_err(|e| e.to_string())?; 560 return Ok(()); 561 } 562 } 563 564 pub struct ArchiveFile { 565 archive_path: PathBuf, 566 archive_name: String, 567 archive_type: ArchiveType, 568 } 569 570 impl ArchiveFile { 571 pub fn new(archive_path: &PathBuf) -> Self { 572 info!("archive_path: {:?}", archive_path); 573 //匹配压缩文件类型 574 let archive_name = archive_path.file_name().unwrap().to_str().unwrap(); 575 for (regex, archivetype) in [ 576 (Regex::new(r"^(.+)\.tar\.gz$").unwrap(), ArchiveType::TarGz), 577 (Regex::new(r"^(.+)\.tar\.xz$").unwrap(), ArchiveType::TarXz), 578 (Regex::new(r"^(.+)\.zip$").unwrap(), ArchiveType::Zip), 579 ] { 580 if regex.is_match(archive_name) { 581 return Self { 582 archive_path: archive_path.parent().unwrap().to_path_buf(), 583 archive_name: archive_name.to_string(), 584 archive_type: archivetype, 585 }; 586 } 587 } 588 Self { 589 archive_path: archive_path.parent().unwrap().to_path_buf(), 590 archive_name: archive_name.to_string(), 591 archive_type: ArchiveType::Undefined, 592 } 593 } 594 595 /// @brief 对self.archive_path路径下名为self.archive_name的压缩文件(tar.gz或zip)进行解压缩 596 /// 597 /// 在此函数中进行路径和文件名有效性的判断，如果有效的话就开始解压缩，根据ArchiveType枚举类型来 598 /// 生成不同的命令来对压缩文件进行解压缩，暂时只支持tar.gz和zip格式，并且都是通过调用bash来解压缩 599 /// 没有引入第三方rust库 600 /// 601 /// 602 /// @return 根据结果返回OK或Err 603 604 pub fn unzip(&self) -> Result<(), String> { 605 let path = &self.archive_path; 606 if !path.is_dir() { 607 return Err(format!("Archive directory {:?} is wrong", path)); 608 } 609 if !path.join(&self.archive_name).is_file() { 610 return Err(format!( 611 " {:?} is not a file", 612 path.join(&self.archive_name) 613 )); 614 } 615 //根据压缩文件的类型生成cmd指令 616 match &self.archive_type { 617 ArchiveType::TarGz | ArchiveType::TarXz => { 618 let mut cmd = Command::new("tar"); 619 cmd.arg("-xf").arg(&self.archive_name); 620 let proc: std::process::Child = cmd 621 .current_dir(path) 622 .stderr(Stdio::piped()) 623 .stdout(Stdio::inherit()) 624 .spawn() 625 .map_err(|e| e.to_string())?; 626 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 627 if !output.status.success() { 628 return Err(format!( 629 "unzip failed, status: {:?}, stderr: {:?}", 630 output.status, 631 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 632 )); 633 } 634 } 635 636 ArchiveType::Zip => { 637 let file = File::open(&self.archive_path.join(&self.archive_name)) 638 .map_err(|e| e.to_string())?; 639 let mut archive = ZipArchive::new(file).map_err(|e| e.to_string())?; 640 for i in 0..archive.len() { 641 let mut file = archive.by_index(i).map_err(|e| e.to_string())?; 642 let outpath = match file.enclosed_name() { 643 Some(path) => self.archive_path.join(path), 644 None => continue, 645 }; 646 if (*file.name()).ends_with('/') { 647 std::fs::create_dir_all(&outpath).map_err(|e| e.to_string())?; 648 } else { 649 if let Some(p) = outpath.parent() { 650 if !p.exists() { 651 std::fs::create_dir_all(&p).map_err(|e| e.to_string())?; 652 } 653 } 654 let mut outfile = File::create(&outpath).map_err(|e| e.to_string())?; 655 std::io::copy(&mut file, &mut outfile).map_err(|e| e.to_string())?; 656 } 657 //设置解压后权限，在Linux中Unzip会丢失权限 658 #[cfg(unix)] 659 { 660 if let Some(mode) = file.unix_mode() { 661 std::fs::set_permissions( 662 &outpath, 663 std::fs::Permissions::from_mode(mode), 664 ) 665 .map_err(|e| e.to_string())?; 666 } 667 } 668 } 669 } 670 _ => { 671 return Err("unsupported archive type".to_string()); 672 } 673 } 674 //删除下载的压缩包 675 info!("unzip successfully, removing archive "); 676 std::fs::remove_file(path.join(&self.archive_name)).map_err(|e| e.to_string())?; 677 //从解压的文件夹中提取出文件并删除下载的压缩包等价于指令"cd *;mv ./* ../../" 678 for entry in path.read_dir().map_err(|e| e.to_string())? { 679 let entry = entry.map_err(|e| e.to_string())?; 680 let path = entry.path(); 681 FileUtils::move_files(&path, &self.archive_path.parent().unwrap()) 682 .map_err(|e| e.to_string())?; 683 //删除空的单独文件夹 684 std::fs::remove_dir_all(&path).map_err(|e| e.to_string())?; 685 } 686 return Ok(()); 687 } 688 } 689 690 pub enum ArchiveType { 691 TarGz, 692 TarXz, 693 Zip, 694 Undefined, 695 } 696