1 use log::info; 2 use regex::Regex; 3 use reqwest::Url; 4 use serde::{Deserialize, Serialize}; 5 use std::os::unix::fs::PermissionsExt; 6 use std::{ 7 fs::File, 8 path::PathBuf, 9 process::{Command, Stdio}, 10 }; 11 use zip::ZipArchive; 12 13 use crate::utils::{file::FileUtils, stdio::StdioUtils}; 14 15 use super::cache::CacheDir; 16 17 /// # Git源 18 /// 19 /// 从Git仓库获取源码 20 #[derive(Debug, Clone, Serialize, Deserialize)] 21 pub struct GitSource { 22 /// Git仓库地址 23 url: String, 24 /// 分支（可选，如果为空，则拉取master）branch和revision只能同时指定一个 25 branch: Option<String>, 26 /// 特定的提交的hash值（可选，如果为空，则拉取branch的最新提交） 27 revision: Option<String>, 28 } 29 30 impl GitSource { 31 pub fn new(url: String, branch: Option<String>, revision: Option<String>) -> Self { 32 Self { 33 url, 34 branch, 35 revision, 36 } 37 } 38 /// # 验证参数合法性 39 /// 40 /// 仅进行形式校验，不会检查Git仓库是否存在，以及分支是否存在、是否有权限访问等 41 pub fn validate(&mut self) -> Result<(), String> { 42 if self.url.is_empty() { 43 return Err("url is empty".to_string()); 44 } 45 // branch和revision不能同时为空 46 if self.branch.is_none() && self.revision.is_none() { 47 self.branch = Some("master".to_string()); 48 } 49 // branch和revision只能同时指定一个 50 if self.branch.is_some() && self.revision.is_some() { 51 return Err("branch and revision are both specified".to_string()); 52 } 53 54 if self.branch.is_some() { 55 if self.branch.as_ref().unwrap().is_empty() { 56 return Err("branch is empty".to_string()); 57 } 58 } 59 if self.revision.is_some() { 60 if self.revision.as_ref().unwrap().is_empty() { 61 return Err("revision is empty".to_string()); 62 } 63 } 64 return Ok(()); 65 } 66 67 pub fn trim(&mut self) { 68 self.url = self.url.trim().to_string(); 69 if let Some(branch) = &mut self.branch { 70 *branch = branch.trim().to_string(); 71 } 72 73 if let Some(revision) = &mut self.revision { 74 *revision = revision.trim().to_string(); 75 } 76 } 77 78 /// # 确保Git仓库已经克隆到指定目录，并且切换到指定分支/Revision 79 /// 80 /// 如果目录不存在，则会自动创建 81 /// 82 /// ## 参数 83 /// 84 /// - `target_dir` - 目标目录 85 /// 86 /// ## 返回 87 /// 88 /// - `Ok(())` - 成功 89 /// - `Err(String)` - 失败，错误信息 90 pub fn prepare(&self, target_dir: &CacheDir) -> Result<(), String> { 91 info!( 92 "Preparing git repo: {}, branch: {:?}, revision: {:?}", 93 self.url, self.branch, self.revision 94 ); 95 96 target_dir.create().map_err(|e| { 97 format!( 98 "Failed to create target dir: {}, message: {e:?}", 99 target_dir.path.display() 100 ) 101 })?; 102 103 if target_dir.is_empty().map_err(|e| { 104 format!( 105 "Failed to check if target dir is empty: {}, message: {e:?}", 106 target_dir.path.display() 107 ) 108 })? { 109 info!("Target dir is empty, cloning repo"); 110 self.clone_repo(target_dir)?; 111 } 112 113 self.checkout(target_dir)?; 114 115 self.pull(target_dir)?; 116 117 return Ok(()); 118 } 119 120 fn check_repo(&self, target_dir: &CacheDir) -> Result<bool, String> { 121 let path: &PathBuf = &target_dir.path; 122 let mut cmd = Command::new("git"); 123 cmd.arg("remote").arg("get-url").arg("origin"); 124 125 // 设置工作目录 126 cmd.current_dir(path); 127 128 // 创建子进程，执行命令 129 let proc: std::process::Child = cmd 130 .stderr(Stdio::piped()) 131 .stdout(Stdio::piped()) 132 .spawn() 133 .map_err(|e| e.to_string())?; 134 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 135 136 if output.status.success() { 137 let mut r = String::from_utf8(output.stdout).unwrap(); 138 r.pop(); 139 Ok(r == self.url) 140 } else { 141 return Err(format!( 142 "git remote get-url origin failed, status: {:?}, stderr: {:?}", 143 output.status, 144 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 145 )); 146 } 147 } 148 149 fn set_url(&self, target_dir: &CacheDir) -> Result<(), String> { 150 let path: &PathBuf = &target_dir.path; 151 let mut cmd = Command::new("git"); 152 cmd.arg("remote") 153 .arg("set-url") 154 .arg("origin") 155 .arg(self.url.as_str()); 156 157 // 设置工作目录 158 cmd.current_dir(path); 159 160 // 创建子进程，执行命令 161 let proc: std::process::Child = cmd 162 .stderr(Stdio::piped()) 163 .spawn() 164 .map_err(|e| e.to_string())?; 165 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 166 167 if !output.status.success() { 168 return Err(format!( 169 "git remote set-url origin failed, status: {:?}, stderr: {:?}", 170 output.status, 171 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 172 )); 173 } 174 Ok(()) 175 } 176 177 fn checkout(&self, target_dir: &CacheDir) -> Result<(), String> { 178 // 确保目标目录中的仓库为所指定仓库 179 if !self.check_repo(target_dir).map_err(|e| { 180 format!( 181 "Failed to check repo: {}, message: {e:?}", 182 target_dir.path.display() 183 ) 184 })? { 185 info!("Target dir isn't specified repo, change remote url"); 186 self.set_url(target_dir)?; 187 } 188 189 let do_checkout = || -> Result<(), String> { 190 let mut cmd = Command::new("git"); 191 cmd.current_dir(&target_dir.path); 192 cmd.arg("checkout"); 193 194 if let Some(branch) = &self.branch { 195 cmd.arg(branch); 196 } 197 if let Some(revision) = &self.revision { 198 cmd.arg(revision); 199 } 200 201 // 强制切换分支，且安静模式 202 cmd.arg("-f").arg("-q"); 203 204 // 创建子进程，执行命令 205 let proc: std::process::Child = cmd 206 .stderr(Stdio::piped()) 207 .spawn() 208 .map_err(|e| e.to_string())?; 209 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 210 211 if !output.status.success() { 212 return Err(format!( 213 "Failed to checkout {}, message: {}", 214 target_dir.path.display(), 215 String::from_utf8_lossy(&output.stdout) 216 )); 217 } 218 return Ok(()); 219 }; 220 221 if let Err(_) = do_checkout() { 222 // 如果切换分支失败，则尝试重新fetch 223 if self.revision.is_some() { 224 self.set_fetch_config(target_dir)?; 225 self.unshallow(target_dir)? 226 }; 227 228 self.fetch_all(target_dir).ok(); 229 do_checkout()?; 230 } 231 232 return Ok(()); 233 } 234 235 pub fn clone_repo(&self, cache_dir: &CacheDir) -> Result<(), String> { 236 let path: &PathBuf = &cache_dir.path; 237 let mut cmd = Command::new("git"); 238 cmd.arg("clone").arg(&self.url).arg(".").arg("--recursive"); 239 240 if let Some(branch) = &self.branch { 241 cmd.arg("--branch").arg(branch).arg("--depth").arg("1"); 242 } 243 244 // 对于克隆，如果指定了revision，则直接克隆整个仓库，稍后再切换到指定的revision 245 246 // 设置工作目录 247 cmd.current_dir(path); 248 249 // 创建子进程，执行命令 250 let proc: std::process::Child = cmd 251 .stderr(Stdio::piped()) 252 .stdout(Stdio::inherit()) 253 .spawn() 254 .map_err(|e| e.to_string())?; 255 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 256 257 if !output.status.success() { 258 return Err(format!( 259 "clone git repo failed, status: {:?}, stderr: {:?}", 260 output.status, 261 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 262 )); 263 } 264 return Ok(()); 265 } 266 267 /// 设置fetch所有分支 268 fn set_fetch_config(&self, target_dir: &CacheDir) -> Result<(), String> { 269 let mut cmd = Command::new("git"); 270 cmd.current_dir(&target_dir.path); 271 cmd.arg("config") 272 .arg("remote.origin.fetch") 273 .arg("+refs/heads/*:refs/remotes/origin/*"); 274 275 // 创建子进程，执行命令 276 let proc: std::process::Child = cmd 277 .stderr(Stdio::piped()) 278 .spawn() 279 .map_err(|e| e.to_string())?; 280 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 281 282 if !output.status.success() { 283 return Err(format!( 284 "Failed to set fetch config {}, message: {}", 285 target_dir.path.display(), 286 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 287 )); 288 } 289 return Ok(()); 290 } 291 /// # 把浅克隆的仓库变成深克隆 292 fn unshallow(&self, target_dir: &CacheDir) -> Result<(), String> { 293 if self.is_shallow(target_dir)? == false { 294 return Ok(()); 295 } 296 297 let mut cmd = Command::new("git"); 298 cmd.current_dir(&target_dir.path); 299 cmd.arg("fetch").arg("--unshallow"); 300 301 cmd.arg("-f"); 302 303 // 创建子进程，执行命令 304 let proc: std::process::Child = cmd 305 .stderr(Stdio::piped()) 306 .spawn() 307 .map_err(|e| e.to_string())?; 308 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 309 310 if !output.status.success() { 311 return Err(format!( 312 "Failed to unshallow {}, message: {}", 313 target_dir.path.display(), 314 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 315 )); 316 } 317 return Ok(()); 318 } 319 320 /// 判断当前仓库是否是浅克隆 321 fn is_shallow(&self, target_dir: &CacheDir) -> Result<bool, String> { 322 let mut cmd = Command::new("git"); 323 cmd.current_dir(&target_dir.path); 324 cmd.arg("rev-parse").arg("--is-shallow-repository"); 325 326 let proc: std::process::Child = cmd 327 .stderr(Stdio::piped()) 328 .spawn() 329 .map_err(|e| e.to_string())?; 330 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 331 332 if !output.status.success() { 333 return Err(format!( 334 "Failed to check if shallow {}, message: {}", 335 target_dir.path.display(), 336 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 337 )); 338 } 339 340 let is_shallow = String::from_utf8_lossy(&output.stdout).trim() == "true"; 341 return Ok(is_shallow); 342 } 343 344 fn fetch_all(&self, target_dir: &CacheDir) -> Result<(), String> { 345 self.set_fetch_config(target_dir)?; 346 let mut cmd = Command::new("git"); 347 cmd.current_dir(&target_dir.path); 348 cmd.arg("fetch").arg("--all"); 349 350 // 安静模式 351 cmd.arg("-f").arg("-q"); 352 353 // 创建子进程，执行命令 354 let proc: std::process::Child = cmd 355 .stderr(Stdio::piped()) 356 .spawn() 357 .map_err(|e| e.to_string())?; 358 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 359 360 if !output.status.success() { 361 return Err(format!( 362 "Failed to fetch all {}, message: {}", 363 target_dir.path.display(), 364 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 365 )); 366 } 367 368 return Ok(()); 369 } 370 371 fn pull(&self, target_dir: &CacheDir) -> Result<(), String> { 372 // 如果没有指定branch，则不执行pull 373 if !self.branch.is_some() { 374 return Ok(()); 375 } 376 info!("git pulling: {}", target_dir.path.display()); 377 378 let mut cmd = Command::new("git"); 379 cmd.current_dir(&target_dir.path); 380 cmd.arg("pull"); 381 382 // 安静模式 383 cmd.arg("-f").arg("-q"); 384 385 // 创建子进程，执行命令 386 let proc: std::process::Child = cmd 387 .stderr(Stdio::piped()) 388 .spawn() 389 .map_err(|e| e.to_string())?; 390 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 391 392 // 如果pull失败，且指定了branch，则报错 393 if !output.status.success() { 394 return Err(format!( 395 "Failed to pull {}, message: {}", 396 target_dir.path.display(), 397 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 398 )); 399 } 400 401 return Ok(()); 402 } 403 } 404 405 /// # 本地源 406 #[derive(Debug, Clone, Serialize, Deserialize)] 407 pub struct LocalSource { 408 /// 本地目录/文件的路径 409 path: PathBuf, 410 } 411 412 impl LocalSource { 413 #[allow(dead_code)] 414 pub fn new(path: PathBuf) -> Self { 415 Self { path } 416 } 417 418 pub fn validate(&self, expect_file: Option<bool>) -> Result<(), String> { 419 if !self.path.exists() { 420 return Err(format!("path {:?} not exists", self.path)); 421 } 422 423 if let Some(expect_file) = expect_file { 424 if expect_file && !self.path.is_file() { 425 return Err(format!("path {:?} is not a file", self.path)); 426 } 427 428 if !expect_file && !self.path.is_dir() { 429 return Err(format!("path {:?} is not a directory", self.path)); 430 } 431 } 432 433 return Ok(()); 434 } 435 436 pub fn trim(&mut self) {} 437 438 pub fn path(&self) -> &PathBuf { 439 &self.path 440 } 441 } 442 443 /// # 在线压缩包源 444 #[derive(Debug, Clone, Serialize, Deserialize)] 445 pub struct ArchiveSource { 446 /// 压缩包的URL 447 url: String, 448 } 449 450 impl ArchiveSource { 451 #[allow(dead_code)] 452 pub fn new(url: String) -> Self { 453 Self { url } 454 } 455 pub fn validate(&self) -> Result<(), String> { 456 if self.url.is_empty() { 457 return Err("url is empty".to_string()); 458 } 459 460 // 判断是一个网址 461 if let Ok(url) = Url::parse(&self.url) { 462 if url.scheme() != "http" && url.scheme() != "https" { 463 return Err(format!("url {:?} is not a http/https url", self.url)); 464 } 465 } else { 466 return Err(format!("url {:?} is not a valid url", self.url)); 467 } 468 return Ok(()); 469 } 470 471 pub fn trim(&mut self) { 472 self.url = self.url.trim().to_string(); 473 } 474 475 /// @brief 下载压缩包并把其中的文件提取至target_dir目录下 476 /// 477 ///从URL中下载压缩包到临时文件夹 target_dir/DRAGONOS_ARCHIVE_TEMP 后 478 ///原地解压，提取文件后删除下载的压缩包。如果 target_dir 非空，就直接使用 479 ///其中内容，不进行重复下载和覆盖 480 /// 481 /// @param target_dir 文件缓存目录 482 /// 483 /// @return 根据结果返回OK或Err 484 pub fn download_unzip(&self, target_dir: &CacheDir) -> Result<(), String> { 485 let url = Url::parse(&self.url).unwrap(); 486 let archive_name = url.path_segments().unwrap().last().unwrap(); 487 let path = &(target_dir.path.join("DRAGONOS_ARCHIVE_TEMP")); 488 //如果source目录没有临时文件夹，且不为空，说明之前成功执行过一次，那么就直接使用之前的缓存 489 if !path.exists() 490 && !target_dir.is_empty().map_err(|e| { 491 format!( 492 "Failed to check if target dir is empty: {}, message: {e:?}", 493 target_dir.path.display() 494 ) 495 })? 496 { 497 //如果source文件夹非空，就直接使用，不再重复下载压缩文件，这里可以考虑加入交互 498 info!("Source files already exist. Using previous source file cache. You should clean {:?} before re-download the archive ", target_dir.path); 499 return Ok(()); 500 } 501 502 if path.exists() { 503 std::fs::remove_dir_all(path).map_err(|e| e.to_string())?; 504 } 505 //创建临时目录 506 std::fs::create_dir(path).map_err(|e| e.to_string())?; 507 info!("downloading {:?}", archive_name); 508 FileUtils::download_file(&self.url, path).map_err(|e| e.to_string())?; 509 //下载成功，开始尝试解压 510 info!("download {:?} finished, start unzip", archive_name); 511 let archive_file = ArchiveFile::new(&path.join(archive_name)); 512 archive_file.unzip()?; 513 //删除创建的临时文件夹 514 std::fs::remove_dir_all(path).map_err(|e| e.to_string())?; 515 return Ok(()); 516 } 517 } 518 519 pub struct ArchiveFile { 520 archive_path: PathBuf, 521 archive_name: String, 522 archive_type: ArchiveType, 523 } 524 525 impl ArchiveFile { 526 pub fn new(archive_path: &PathBuf) -> Self { 527 info!("archive_path: {:?}", archive_path); 528 //匹配压缩文件类型 529 let archive_name = archive_path.file_name().unwrap().to_str().unwrap(); 530 for (regex, archivetype) in [ 531 (Regex::new(r"^(.+)\.tar\.gz$").unwrap(), ArchiveType::TarGz), 532 (Regex::new(r"^(.+)\.tar\.xz$").unwrap(), ArchiveType::TarXz), 533 (Regex::new(r"^(.+)\.zip$").unwrap(), ArchiveType::Zip), 534 ] { 535 if regex.is_match(archive_name) { 536 return Self { 537 archive_path: archive_path.parent().unwrap().to_path_buf(), 538 archive_name: archive_name.to_string(), 539 archive_type: archivetype, 540 }; 541 } 542 } 543 Self { 544 archive_path: archive_path.parent().unwrap().to_path_buf(), 545 archive_name: archive_name.to_string(), 546 archive_type: ArchiveType::Undefined, 547 } 548 } 549 550 /// @brief 对self.archive_path路径下名为self.archive_name的压缩文件(tar.gz或zip)进行解压缩 551 /// 552 /// 在此函数中进行路径和文件名有效性的判断，如果有效的话就开始解压缩，根据ArchiveType枚举类型来 553 /// 生成不同的命令来对压缩文件进行解压缩，暂时只支持tar.gz和zip格式，并且都是通过调用bash来解压缩 554 /// 没有引入第三方rust库 555 /// 556 /// 557 /// @return 根据结果返回OK或Err 558 559 pub fn unzip(&self) -> Result<(), String> { 560 let path = &self.archive_path; 561 if !path.is_dir() { 562 return Err(format!("Archive directory {:?} is wrong", path)); 563 } 564 if !path.join(&self.archive_name).is_file() { 565 return Err(format!( 566 " {:?} is not a file", 567 path.join(&self.archive_name) 568 )); 569 } 570 //根据压缩文件的类型生成cmd指令 571 match &self.archive_type { 572 ArchiveType::TarGz | ArchiveType::TarXz => { 573 let mut cmd = Command::new("tar"); 574 cmd.arg("-xf").arg(&self.archive_name); 575 let proc: std::process::Child = cmd 576 .current_dir(path) 577 .stderr(Stdio::piped()) 578 .stdout(Stdio::inherit()) 579 .spawn() 580 .map_err(|e| e.to_string())?; 581 let output = proc.wait_with_output().map_err(|e| e.to_string())?; 582 if !output.status.success() { 583 return Err(format!( 584 "unzip failed, status: {:?}, stderr: {:?}", 585 output.status, 586 StdioUtils::tail_n_str(StdioUtils::stderr_to_lines(&output.stderr), 5) 587 )); 588 } 589 } 590 591 ArchiveType::Zip => { 592 let file = File::open(&self.archive_path.join(&self.archive_name)) 593 .map_err(|e| e.to_string())?; 594 let mut archive = ZipArchive::new(file).map_err(|e| e.to_string())?; 595 for i in 0..archive.len() { 596 let mut file = archive.by_index(i).map_err(|e| e.to_string())?; 597 let outpath = match file.enclosed_name() { 598 Some(path) => self.archive_path.join(path), 599 None => continue, 600 }; 601 if (*file.name()).ends_with('/') { 602 std::fs::create_dir_all(&outpath).map_err(|e| e.to_string())?; 603 } else { 604 if let Some(p) = outpath.parent() { 605 if !p.exists() { 606 std::fs::create_dir_all(&p).map_err(|e| e.to_string())?; 607 } 608 } 609 let mut outfile = File::create(&outpath).map_err(|e| e.to_string())?; 610 std::io::copy(&mut file, &mut outfile).map_err(|e| e.to_string())?; 611 } 612 //设置解压后权限，在Linux中Unzip会丢失权限 613 #[cfg(unix)] 614 { 615 if let Some(mode) = file.unix_mode() { 616 std::fs::set_permissions( 617 &outpath, 618 std::fs::Permissions::from_mode(mode), 619 ) 620 .map_err(|e| e.to_string())?; 621 } 622 } 623 } 624 } 625 _ => { 626 return Err("unsupported archive type".to_string()); 627 } 628 } 629 //删除下载的压缩包 630 info!("unzip successfully, removing archive "); 631 std::fs::remove_file(path.join(&self.archive_name)).map_err(|e| e.to_string())?; 632 //从解压的文件夹中提取出文件并删除下载的压缩包等价于指令"cd *;mv ./* ../../" 633 for entry in path.read_dir().map_err(|e| e.to_string())? { 634 let entry = entry.map_err(|e| e.to_string())?; 635 let path = entry.path(); 636 FileUtils::move_files(&path, &self.archive_path.parent().unwrap()) 637 .map_err(|e| e.to_string())?; 638 //删除空的单独文件夹 639 std::fs::remove_dir_all(&path).map_err(|e| e.to_string())?; 640 } 641 return Ok(()); 642 } 643 } 644 645 pub enum ArchiveType { 646 TarGz, 647 TarXz, 648 Zip, 649 Undefined, 650 } 651